EPP analysis on-par with old analysis.

This commit is a transition point that makes some assumptions about new
callbacks for the compiler modules, which will likely not hold when it
comes to making epp be able to read and handle multiple applications at
once (so it can resolve parse transforms and behaviours across OTP
apps).

As such, a follow-up commit is going to be completing that one and
changing its API in incompatible ways; if you find this commit during a
bisect, it's probably not a good one in which to run a bisection.

bootstrap

@@ -19,6 +19,9 @@ main(_) ->
     %% manages to discover those in _build/prod from previous builds and
     %% cause weird failures when compilers get modified between releases.
     rm_rf("_build/prod"),
+    %% The same pattern happens with default/ as well, particularly when
+    %% developig new things.
+    rm_rf("_build/default"),
 
     %% We fetch a few deps from hex for boostraping,
     %% so we must compile r3_safe_erl_term.xrl which

src/rebar_compiler.erl

@@ -15,10 +15,11 @@
 -type extension() :: string().
 -type out_mappings() :: [{extension(), file:filename()}].
 
--callback context(rebar_app_info:t()) -> #{src_dirs     => [file:dirname()],
-                                           include_dirs => [file:dirname()],
-                                           src_ext      => extension(),
-                                           out_mappings => out_mappings()}.
+-callback context(rebar_app_info:t()) -> #{src_dirs     := [file:dirname()],
+                                           include_dirs := [file:dirname()],
+                                           src_ext      := extension(),
+                                           out_mappings := out_mappings(),
+                                           dependencies_opts => term()}.
 -callback needed_files(digraph:graph(), [file:filename()], out_mappings(),
                        rebar_app_info:t()) ->
     {{[file:filename()], term()}, % ErlFirstFiles (erl_opts global priority)
@@ -26,17 +27,19 @@
       {[file:filename()], [file:filename()]}, % {Sequential, Parallel}
       term()}}.
 -callback dependencies(file:filename(), file:dirname(), [file:dirname()]) -> [file:filename()].
+-callback dependencies(file:filename(), file:dirname(), [file:dirname()], term()) -> [file:filename()].
 -callback compile(file:filename(), out_mappings(), rebar_dict(), list()) ->
     ok | {ok, [string()]} | {ok, [string()], [string()]}.
 -callback clean([file:filename()], rebar_app_info:t()) -> _.
 
+-optional_callbacks([dependencies/4]).
 
 -define(RE_PREFIX, "^(?!\\._)").
 
 -spec compile_all([{module(), digraph:graph()}, ...], rebar_app_info:t()) -> ok
       ;          ([module(), ...], rebar_app_info:t()) -> ok.
 compile_all(DAGs, AppInfo) when is_tuple(hd(DAGs)) -> % > 3.13.0
-    prepare_compiler_env(AppInfo),
+    prepare_compiler_env(DAGs, AppInfo),
     lists:foreach(fun({Compiler, G}) ->
         run(G, Compiler, AppInfo),
         %% TODO: disable default recursivity in extra_src_dirs compiling to
@@ -58,7 +61,7 @@ compile_all(Compilers, AppInfo) -> % =< 3.13.0 interface; plugins use this!
         rebar_compiler_dag:terminate(G)
      end, Compilers).
 
-prepare_compiler_env(AppInfo) ->
+prepare_compiler_env(DAGs, AppInfo) ->
     EbinDir = rebar_utils:to_list(rebar_app_info:ebin_dir(AppInfo)),
     %% Make sure that outdir is on the path
     ok = rebar_file_utils:ensure_dir(EbinDir),
@@ -68,13 +71,16 @@ prepare_compiler_env(AppInfo) ->
     %% called here for clarity as it's required by both opts_changed/2
     %% and erl_compiler_opts_set/0 in needed_files
     _ = code:ensure_loaded(compile),
+    [code:ensure_loaded(Mod) || {Mod, _} <- DAGs],
     ok.
 
 run(G, CompilerMod, AppInfo) ->
+    Ctx = CompilerMod:context(AppInfo),
     #{src_dirs := SrcDirs,
       include_dirs := InclDirs,
       src_ext := SrcExt,
-      out_mappings := Mappings} = CompilerMod:context(AppInfo),
+      out_mappings := Mappings,
+      dependencies_opts := DepOpts} = maps:merge(default_ctx(), Ctx),
 
     BaseDir = rebar_utils:to_list(rebar_app_info:dir(AppInfo)),
     EbinDir = rebar_utils:to_list(rebar_app_info:ebin_dir(AppInfo)),
@@ -88,7 +94,7 @@ run(G, CompilerMod, AppInfo) ->
     InDirs = lists:usort(AbsInclDirs ++ AbsSrcDirs),
 
     rebar_compiler_dag:prune(G, AbsSrcDirs, EbinDir, FoundFiles),
-    rebar_compiler_dag:update(G, CompilerMod, InDirs, FoundFiles),
+    rebar_compiler_dag:update(G, CompilerMod, InDirs, FoundFiles, DepOpts),
     {{FirstFiles, FirstFileOpts},
      {RestFiles, Opts}} = CompilerMod:needed_files(G, FoundFiles, Mappings, AppInfo),
 
@@ -253,3 +259,6 @@ add_to_includes(AppInfo, Dirs) ->
     NewErlOpts = [{i, Dir} || Dir <- Dirs] ++ List,
     NewOpts = rebar_opts:set(Opts, erl_opts, NewErlOpts),
     rebar_app_info:opts(AppInfo, NewOpts).
+
+default_ctx() ->
+    #{dependencies_opts => []}.

src/rebar_compiler_dag.erl

@@ -1,7 +1,7 @@
 %%% Module handling the directed graph required for the analysis
 %%% of all top-level applications by the various compiler plugins.
 -module(rebar_compiler_dag).
--export([init/4, prune/4, update/4, maybe_store/5, terminate/1]).
+-export([init/4, prune/4, update/5, maybe_store/5, terminate/1]).
 
 -include("rebar.hrl").
 
@@ -59,10 +59,11 @@ prune(G, SrcDirs, EbinDir, Erls) ->
 %% to propagate file changes.
 %%
 %% To be replaced by a more declarative EPP-based flow.
--spec update(dag(), module(), [file:filename_all()], [file:filename_all()]) -> ok.
-update(_, _, _, []) ->
+-spec update(dag(), module(), [file:filename_all()], [file:filename_all()],
+             term()) -> ok.
+update(_, _, _, [], _) ->
     ok;
-update(G, Compiler, InDirs, [Source|Erls]) ->
+update(G, Compiler, InDirs, [Source|Erls], DepOpts) ->
     case digraph:vertex(G, Source) of
         {_, LastUpdated} ->
             case filelib:last_modified(Source) of
@@ -72,25 +73,27 @@ update(G, Compiler, InDirs, [Source|Erls]) ->
                     %% All the edges will be erased automatically.
                     digraph:del_vertex(G, Source),
                     mark_dirty(G),
-                    update(G, Compiler, InDirs, Erls);
+                    update(G, Compiler, InDirs, Erls, DepOpts);
                 LastModified when LastUpdated < LastModified ->
-                    add_to_dag(G, Compiler, InDirs, Source, LastModified, filename:dirname(Source)),
-                    update(G, Compiler, InDirs, Erls);
+                    add_to_dag(G, Compiler, InDirs, Source, LastModified,
+                               filename:dirname(Source), DepOpts),
+                    update(G, Compiler, InDirs, Erls, DepOpts);
                 _ ->
                     AltErls = digraph:out_neighbours(G, Source),
                     %% Deps must be explored before the module itself
-                    update(G, Compiler, InDirs, AltErls),
+                    update(G, Compiler, InDirs, AltErls, DepOpts),
                     Modified = is_dirty(G),
                     MaxModified = update_max_modified_deps(G, Source),
                     case Modified orelse MaxModified > LastUpdated of
                         true -> mark_dirty(G);
                         false -> ok
                     end,
-                    update(G, Compiler, InDirs, Erls)
+                    update(G, Compiler, InDirs, Erls, DepOpts)
             end;
         false ->
-            add_to_dag(G, Compiler, InDirs, Source, filelib:last_modified(Source), filename:dirname(Source)),
-            update(G, Compiler, InDirs, Erls)
+            add_to_dag(G, Compiler, InDirs, Source, filelib:last_modified(Source),
+                       filename:dirname(Source), DepOpts),
+            update(G, Compiler, InDirs, Erls, DepOpts)
     end.
 
 maybe_store(G, Dir, Compiler, Label, CritMeta) ->
@@ -165,13 +168,18 @@ target_base(OutDir, Source) ->
 
 %% @private a file has been found to change or wasn't part of the DAG before,
 %% and must be added, along with all its dependencies.
-add_to_dag(G, Compiler, InDirs, Source, LastModified, SourceDir) ->
-    AbsIncls = Compiler:dependencies(Source, SourceDir, InDirs),
+add_to_dag(G, Compiler, InDirs, Source, LastModified, SourceDir, DepOpts) ->
+    AbsIncls = case erlang:function_exported(Compiler, dependencies, 4) of
+        false ->
+            Compiler:dependencies(Source, SourceDir, InDirs);
+        true ->
+            Compiler:dependencies(Source, SourceDir, InDirs, DepOpts)
+    end,
     digraph:add_vertex(G, Source, LastModified),
     digraph:del_edges(G, digraph:out_edges(G, Source)),
     %% Deps must be explored before the module itself
     [begin
-         update(G, Compiler, InDirs, [Incl]),
+         update(G, Compiler, InDirs, [Incl], DepOpts),
          digraph:add_edge(G, Source, Incl)
      end || Incl <- AbsIncls],
     mark_dirty(G),

src/rebar_compiler_epp.erl

@@ -0,0 +1,126 @@
+%%% @doc
+%%% Analyze erlang-related files and compilation data using EPP, in order to
+%%% build complete and accurate DAGs
+%%% @end
+-module(rebar_compiler_epp).
+-export([deps/2, resolve_module/2]).
+-include_lib("kernel/include/file.hrl").
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%% Basic File Handling %%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+%% Find all Erlang code dependencies for a given file
+-spec deps(file:filename_all(), Opts) -> Attributes when
+      Opts :: [Opt, ...],
+      Opt :: {includes, [file:filename_all()]}
+           | {macros, [file:filename_all()]},
+      Attributes :: #{include := [file:filename_all()],
+                      missing_include_file := [file:filename_all()],
+                      missing_include_lib := [file:filename_all()],
+                      behaviour := [atom()],
+                      parse_transform := [atom()],
+                      is_behaviour := boolean()}.
+deps(File, Opts) ->
+    {ok, Forms} = epp:parse_file(File, Opts),
+    normalize(handle_forms(Forms, default_attrs())).
+
+%% Find the path matching a given erlang module
+resolve_module(Mod, Paths) ->
+    ModStr = atom_to_list(Mod),
+    try
+        [throw(P) || P <- Paths, ModStr =:= filename:basename(P, ".erl")],
+        {error, not_found}
+    catch
+        Path -> {ok, Path}
+    end.
+
+%%%%%%%%%%%%%%%
+%%% PRIVATE %%%
+%%%%%%%%%%%%%%%
+
+default_attrs() ->
+    #{include => [],
+      missing_include_file => [],
+      missing_include_lib => [],
+      behaviour => [],
+      parse_transform => [],
+      is_behaviour => false}.
+
+normalize(Map) ->
+    #{include := Incl,
+      missing_include_file := InclF,
+      missing_include_lib := InclL,
+      behaviour := Behaviour,
+      parse_transform := PTrans} = Map,
+    Map#{include => lists:usort(Incl),
+         missing_include_file => lists:usort(InclF),
+         missing_include_lib => lists:usort(InclL),
+         behaviour => lists:usort(Behaviour),
+         parse_transform => lists:usort(PTrans)}.
+
+handle_forms([File|Forms], Map) ->
+    lists:foldl(fun handle_form/2, Map, drop_self_file(File, Forms)).
+
+drop_self_file(_, []) ->
+    [];
+drop_self_file({attribute, _, file, {Path,_}} = File,
+               [{attribute,_, file, {Path,_}} | Rest]) ->
+    drop_self_file(File, Rest);
+drop_self_file(File, [Keep|Rest]) ->
+    [Keep | drop_self_file(File, Rest)].
+
+%% Included files (both libs and direct includes);
+%% There are also references to the module's own file declaration
+%% in there, but this is dropped by `drop_self_file/2' and assumed
+%% to be gone here.
+handle_form({attribute, _Line, file, {Path, Ln}}, Map) ->
+    %% Some people think they're funny and they go include attributes
+    %% like:
+    %%  -file("fake/file.hrl", Ln).
+    %% Which are expanded to the very clause we have here, which in
+    %% turn is impossible to distinguish from actual included files
+    %% once checked through epp. The way we work around that here
+    %% is to check if the path is absolute, and if so, keep it in since
+    %% epp has expanded it; otherwise consider it to be a failed include.
+    %% This is not perfect but we can't do much more without touching the
+    %% disk and hopefully nobody else in the community has relied on this
+    %% thing.
+    case filename:absname(Path) of
+        Path ->
+            maps:update_with(include, fun(L) -> [Path|L] end, [Path], Map);
+        _ -> % argh!
+            handle_form({error, {Ln, {epp, {include, file, Path}}}}, Map)
+    end;
+%% Include files that EPP couldn't resolve
+handle_form({error, {_Line, epp, {include, file, Name}}}, Map) ->
+    maps:update_with(missing_include_file, fun(L) -> [Name|L] end, [Name], Map);
+handle_form({error, {_Line, epp, {include, lib, Path}}}, Map) ->
+    maps:update_with(missing_include_lib, fun(L) -> [Path|L] end, [Path], Map);
+%% Behaviour implementation declaration
+handle_form({attribute, _Line, behaviour, Name}, Map) ->
+    maps:update_with(behaviour, fun(L) -> [Name|L] end, [Name], Map);
+handle_form({attribute, _Line, behavior, Name}, Map) ->
+    maps:update_with(behaviour, fun(L) -> [Name|L] end, [Name], Map);
+%% Extract parse transforms
+handle_form({attribute, Line, compile, Attr}, Map) when not is_list(Attr) ->
+    handle_form({attribute, Line, compile, [Attr]}, Map);
+handle_form({attribute, _Line, compile, Attrs}, Map) ->
+    Mods = [case T of
+                {_, {M,_}} -> M;
+                {_, M} -> M
+            end || T <- proplists:lookup_all(parse_transform, Attrs)],
+    maps:update_with(parse_transform, fun(L) -> Mods++L end, Mods, Map);
+%% Current style behaviour specification declaration
+handle_form({attribute, _Line, callback, _}, Map) ->
+    Map#{is_behaviour => true};
+%% Old style behaviour specification, both spellings supported
+%% The function needs to be exported, but we skip over that logic
+%% for now.
+handle_form({function, _Line, behaviour_info, 1, _}, Map) ->
+    Map#{is_behaviour => true};
+handle_form({function, _Line, behavior_info, 1, _}, Map) ->
+    Map#{is_behaviour => true};
+%% Skip the rest
+handle_form(_, Map) ->
+    Map.

src/rebar_compiler_erl.erl

@@ -4,7 +4,7 @@
 
 -export([context/1,
          needed_files/4,
-         dependencies/3,
+         dependencies/3, dependencies/4,
          compile/4,
          clean/2,
          format_error/1]).
@@ -26,11 +26,18 @@ context(AppInfo) ->
     ErlOpts = rebar_opts:erl_opts(RebarOpts),
     ErlOptIncludes = proplists:get_all_values(i, ErlOpts),
     InclDirs = lists:map(fun(Incl) -> filename:absname(Incl) end, ErlOptIncludes),
+    AbsIncl = [filename:join([OutDir, "include"]) | InclDirs],
+    Macros = [case Tup of
+                  {d,Name} -> Name;
+                  {d,Name,Val} -> {Name,Val}
+              end || Tup <- ErlOpts,
+                     is_tuple(Tup) andalso element(1,Tup) == d],
 
     #{src_dirs => ExistingSrcDirs,
-      include_dirs => [filename:join([OutDir, "include"]) | InclDirs],
+      include_dirs => AbsIncl,
       src_ext => ".erl",
-      out_mappings => Mappings}.
+      out_mappings => Mappings,
+      dependencies_opts => [{includes, AbsIncl}, {macros, Macros}]}.
 
 
 needed_files(Graph, FoundFiles, _, AppInfo) ->
@@ -86,6 +93,32 @@ dependencies(Source, SourceDir, Dirs) ->
             throw(?PRV_ERROR({cannot_read_file, Source, file:format_error(Reason)}))
     end.
 
+dependencies(Source, _SourceDir, Dirs, DepOpts) ->
+    try rebar_compiler_epp:deps(Source, DepOpts) of
+        #{include := AbsIncls,
+          missing_include_file := _MissIncl,
+          missing_include_lib := _MissInclLib,
+          parse_transform := PTrans,
+          behaviour := Behaviours} ->
+            %% TODO: resolve behaviours cross-app
+            %% TODO: resolve parse_transforms cross-app
+            %% TODO: report on missing files
+            %% TODO: check for core transforms?
+            expand_file_names([module_to_erl(Mod) || Mod <- PTrans], Dirs) ++
+            expand_file_names([module_to_erl(Mod) || Mod <- Behaviours], Dirs) ++
+            AbsIncls
+    catch
+        error:{badmatch, {error, Reason}} ->
+            case file:format_error(Reason) of
+                "unknown POSIX error" ->
+                    throw(?PRV_ERROR({cannot_read_file, Source, Reason}));
+                ReadableReason ->
+                    throw(?PRV_ERROR({cannot_read_file, Source, ReadableReason}))
+            end;
+        error:Reason ->
+            throw(?PRV_ERROR({cannot_read_file, Source, Reason}))
+    end.
+
 compile(Source, [{_, OutDir}], Config, ErlOpts) ->
     case compile:file(Source, [{outdir, OutDir} | ErlOpts]) of
         {ok, _Mod} ->

test/rebar_compiler_epp_SUITE.erl

@@ -0,0 +1,300 @@
+%%% @doc
+%%% Unit tests for epp-related compiler utils.
+%%% Make it easier to validate internal behaviour of compiler data and
+%%% handling of module parsing without having to actually set up
+%%% entire projects.
+%%% @end
+-module(rebar_compiler_epp_SUITE).
+-include_lib("common_test/include/ct.hrl").
+-include_lib("eunit/include/eunit.hrl").
+-compile([export_all, nowarn_export_all]).
+
+all() ->
+    [{group, module}].
+
+groups() ->
+    [{module, [], [
+        analyze, analyze_old_behaviour, analyze_old_behavior,
+        analyze_empty, analyze_bad_mod,
+        resolve_module
+     ]}
+    ].
+
+init_per_group(module, Config) ->
+    to_file(Config, {"direct.hrl", "-direct(val). "}),
+    Config;
+init_per_group(_, Config) ->
+    Config.
+
+end_per_group(_, Config) ->
+    Config.
+
+init_per_testcase(_, Config) ->
+    Config.
+
+end_per_testcase(_, Config) ->
+    Config.
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%% module analysis group %%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+analyze() ->
+    [{docs, "Analyzing a module returns all the "
+            "parseable dependencies for it in a map."}].
+analyze(Config) ->
+    ?assert(check_analyze(
+       #{include => [
+           "eunit-[0-9.]+/include/eunit.hrl$",
+           "stdlib-[0-9.]+/include/assert.hrl$",
+           "/direct.hrl$"
+         ],
+         %% missing includes
+         missing_include_file => [
+            "^false.hrl$"
+         ],
+         missing_include_lib => [
+            "^some_app/include/lib.hrl$"
+         ],
+         parse_transform => [
+            erl_id_trans,
+            eunit_autoexport, % added by include file!
+            missing_parse_trans1,
+            missing_parse_trans2
+         ],
+         behaviour => [gen_server, gen_statem],
+         is_behaviour => true
+       },
+       rebar_compiler_epp:deps(
+         to_file(Config, fake_mod()),
+         [{includes, []}, {macros, []}]
+       )
+    )),
+    ok.
+
+analyze_old_behaviour() ->
+    [{docs, "Analyzing old-style behaviour annotation"}].
+analyze_old_behaviour(Config) ->
+    ?assert(check_analyze(
+       #{include => [],
+         missing_include_file => [],
+         missing_include_lib => [],
+         parse_transform => [],
+         behaviour => [],
+         is_behaviour => true
+       },
+       rebar_compiler_epp:deps(
+         to_file(Config, old_behaviour_mod()),
+         [{includes, []}, {macros, []}]
+       )
+    )),
+    ok.
+
+analyze_old_behavior() ->
+    [{docs, "Analyzing old-style behavior annotation"}].
+analyze_old_behavior(Config) ->
+    ?assert(check_analyze(
+       #{include => [],
+         missing_include_file => [],
+         missing_include_lib => [],
+         parse_transform => [],
+         behaviour => [],
+         is_behaviour => true
+       },
+       rebar_compiler_epp:deps(
+         to_file(Config, old_behavior_mod()),
+         [{includes, []}, {macros, []}]
+       )
+    )),
+    ok.
+
+analyze_empty() ->
+    [{docs, "Making sure empty files are properly handled as valid but null "
+            "and let some other compiler phase handle this. We follow "
+            "what EPP handles."}].
+analyze_empty(Config) ->
+    ?assert(check_analyze(
+       #{include => [],
+         missing_include_file => [],
+         missing_include_lib => [],
+         parse_transform => [],
+         behaviour => [],
+         is_behaviour => false
+       },
+       rebar_compiler_epp:deps(
+         to_file(Config, empty_mod()),
+         [{includes, []}, {macros, []}]
+       )
+    )),
+    ok.
+
+analyze_bad_mod() ->
+    [{docs, "Errors for bad modules that don't compile are skipped "
+            "by EPP and so we defer that to a later phase of the "
+            "compilation process"}].
+analyze_bad_mod(Config) ->
+    ?assert(check_analyze(
+       #{include => [],
+         missing_include_file => [],
+         missing_include_lib => [],
+         parse_transform => [],
+         behaviour => [],
+         is_behaviour => false
+       },
+       rebar_compiler_epp:deps(
+         to_file(Config, bad_mod()),
+         [{includes, []}, {macros, []}]
+       )
+    )),
+    ok.
+
+resolve_module() ->
+    [{doc, "given a module name and a bunch of paths, find "
+           "the first path that matches the module"}].
+resolve_module(Config) ->
+    Path1 = to_file(Config, fake_mod()),
+    Path2 = to_file(Config, old_behaviour_mod()),
+    Path3 = to_file(Config, empty_mod()),
+    ?assertEqual(
+       {ok, Path2},
+       rebar_compiler_epp:resolve_module(
+         old_behaviour,
+         [Path1, Path2, Path3]
+       )
+    ),
+    ok.
+
+%%%%%%%%%%%%%%%
+%%% HELPERS %%%
+%%%%%%%%%%%%%%%
+
+%% check each field of `Map' and validate them against `CheckMap'.
+%% This allows to check each value in the map has a matching assertion.
+%% Then check each field of `CheckMap' against `Map' to find if
+%% any missing value exists.
+check_analyze(CheckMap, Map) ->
+    ct:pal("check_analyze:~n~p~n~p", [CheckMap, Map]),
+    maps:fold(fun(K,V,Acc) -> check(CheckMap, K, V) and Acc end,
+              true, Map)
+    andalso
+    maps:fold(
+      fun(K,_,Acc) ->
+          check(CheckMap, K, maps:get(K, Map, make_ref())) and Acc
+      end,
+      true,
+      Map
+    ).
+
+check(Map, K, V) ->
+    case maps:is_key(K, Map) of
+        false -> false;
+        true ->
+            #{K := Val} = Map,
+            compare_val(Val, V)
+    end.
+
+%% two identical values always works
+compare_val(V, V) ->
+    true;
+%% compare lists of strings; each string must be checked individually
+%% because they are assumed to be regexes.
+compare_val(V1, V2) when is_list(hd(V1)) ->
+    match_regexes(V1, V2);
+compare_val(V1, _V2) when not is_integer(hd(V1)) ->
+    %% failing list of some sort, but not a string
+    false;
+%% strings as regexes
+compare_val(V1, V2) when is_list(V1) ->
+    match_regex(V1, [V2]) =/= nomatch;
+%% anything else is not literally the same and is bad
+compare_val(_, _) ->
+    false.
+
+match_regexes([], List) ->
+    List == []; % no extra patterns, that would be weird
+match_regexes([H|T], List) ->
+    case match_regex(H, List) of
+       nomatch ->
+            false;
+       {ok, Entry} ->
+            match_regexes(T, List -- [Entry])
+    end.
+
+match_regex(_Pattern, []) ->
+    nomatch;
+match_regex(Pattern, [H|T]) ->
+    case re:run(H, Pattern) of
+        nomatch -> match_regex(Pattern, T);
+        _ -> {ok, H}
+    end.
+
+%% custom zip function that causes value failures (by using make_ref()
+%% that will never match in compare_val/2) rather than crashing because
+%% of lists of different lengths.
+zip([], []) -> [];
+zip([], [H|T]) -> [{make_ref(),H} | zip([], T)];
+zip([H|T], []) -> [{H,make_ref()} | zip(T, [])];
+zip([X|Xs], [Y|Ys]) -> [{X,Y} | zip(Xs, Ys)].
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%% Module specifications %%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+%% turn a module string to a file that will live in CT's scratch dir
+to_file(Config, {Name,Contents}) ->
+    Path = filename:join([?config(priv_dir, Config), Name]),
+    file:write_file(Path, Contents, [sync]),
+    Path.
+
+%% base module with all the interesting includes and attributes
+%% we want to track
+fake_mod() ->
+    {"somemod.erl", "
+-module(somemod).
+-export([f/1]).
+-include(\"direct.hrl\").
+-include(\"direct.hrl\").
+-include_lib(\"some_app/include/lib.hrl\").
+-include_lib(\"eunit/include/eunit.hrl\").
+-compile({parse_transform, {erl_id_trans, []}}).
+-compile({parse_transform, missing_parse_trans1}).
+-compile([{parse_transform, {missing_parse_trans2, []}}]).
+-behaviour(gen_server).
+-behavior(gen_statem).
+-callback f() -> ok.
+-ifdef(OPT).
+-include(\"true.hrl\").
+-else.
+-include(\"false.hrl\").
+-endif.
+f(X) -> X.
+    "}.
+
+%% variations for attributes that can't be checked in the
+%% same base module
+old_behaviour_mod() ->
+    {"old_behaviour.erl", "
+-module(old_behaviour).
+-export([f/1, behaviour_info/1]).
+f(X) -> X.
+behaviour_info(callbacks) -> [{f,1}].
+    "}.
+
+old_behavior_mod() ->
+    {"old_behaviour.erl", "
+-module(old_behaviour).
+-export([f/1, behaviour_info/1]).
+f(X) -> X.
+behavior_info(callbacks) -> [{f,1}].
+    "}.
+
+empty_mod() ->
+    {"empty.erl", ""}.
+
+bad_mod() ->
+    {"badmod.erl", "
+-module(bad_mod). % wrong name!
+f(x) -> X+1. % bad vars
+f((x)cv) -> bad syntax.
+    "}.
+