Add xref provider for cross reference analysis

* Add a provider for the xref tool for running cross reference
  analysis on a project. Most of the code has been ported directly
  from the rebar2 rebar_xref module with some modification and cleanup
  to support testing.
* Port over the eunit test suite from rebar2, but convert it to
  common_test. The testing is the same, but now the
  erlang term output is examined to determine if the test run is successful
  instead of scanning the console output for the expected strings.

README.md

@@ -43,6 +43,7 @@ limit scope.
 | update     | Update package index |
 | upgrade    | Fetch latest version of dep |
 | version    | Print current version of Erlang/OTP and rebar |
+| xref       | Run cross reference analysis on the project |
 
 ### Commands still to do
 

THANKS

@@ -128,4 +128,5 @@ Alexander Verbitsky
 Andras Horvath
 Drew Varner
 Omar Yasin
-Tristan Sloughter
+Tristan Sloughter
+Kelly McLaughlin

src/rebar.app.src

@@ -43,6 +43,7 @@
                      rebar_prv_release,
                      rebar_prv_version,
                      rebar_prv_common_test,
+                     rebar_prv_xref,
                      rebar_prv_help]}
         ]}
  ]}.

src/rebar_prv_xref.erl

@@ -0,0 +1,296 @@
+%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*-
+%% ex: ts=4 sw=4 et
+
+-module(rebar_prv_xref).
+
+-behaviour(provider).
+
+-export([init/1,
+         do/1,
+         format_error/1]).
+
+-include("rebar.hrl").
+
+-define(PROVIDER, xref).
+-define(DEPS, [compile]).
+-define(SUPPORTED_XREFS, [undefined_function_calls, undefined_functions,
+                          locals_not_used, exports_not_used,
+                          deprecated_function_calls, deprecated_functions]).
+
+%% ===================================================================
+%% Public API
+%% ===================================================================
+
+-spec init(rebar_state:t()) -> {ok, rebar_state:t()}.
+init(State) ->
+    Provider = providers:create([{name, ?PROVIDER},
+                                 {module, ?MODULE},
+                                 {deps, ?DEPS},
+                                 {bare, false},
+                                 {example, "rebar3 xref"},
+                                 {short_desc, short_desc()},
+                                 {desc, desc()}]),
+    State1 = rebar_state:add_provider(State, Provider),
+    {ok, State1}.
+
+-spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}.
+do(State) ->
+    {OriginalPath, XrefChecks} = prepare(State),
+
+    %% Run xref checks
+    ?INFO("Running cross reference analysis...", []),
+    XrefResults = xref_checks(XrefChecks),
+
+    %% Run custom queries
+    QueryChecks = rebar_state:get(State, xref_queries, []),
+    QueryResults = lists:foldl(fun check_query/2, [], QueryChecks),
+
+    ok = cleanup(OriginalPath),
+
+    case XrefResults =:= [] andalso QueryResults =:= [] of
+        true ->
+            {ok, State};
+        false ->
+            {error, {?MODULE, {xref_issues, XrefResults, QueryResults}}}
+    end.
+
+-spec format_error(any()) -> iolist().
+format_error({xref_issues, XrefResults, QueryResults}) ->
+    lists:flatten(display_results(XrefResults, QueryResults));
+format_error(Reason) ->
+    io_lib:format("~p", [Reason]).
+
+%% ===================================================================
+%% Internal functions
+%% ===================================================================
+
+short_desc() ->
+    "Run cross reference analysis".
+
+desc() ->
+    io_lib:format(
+      "~s~n"
+      "~n"
+      "Valid rebar.config options:~n"
+      "  ~p~n"
+      "  ~p~n"
+      "  ~p~n"
+      "  ~p~n",
+      [short_desc(),
+       {xref_warnings, false},
+       {xref_extra_paths,[]},
+       {xref_checks, [undefined_function_calls, undefined_functions,
+                      locals_not_used, exports_not_used,
+                      deprecated_function_calls, deprecated_functions]},
+       {xref_queries,
+        [{"(xc - uc) || (xu - x - b"
+          " - (\"mod\":\".*foo\"/\"4\"))",[]}]}
+      ]).
+
+-spec prepare(rebar_state:t()) -> list(atom()).
+prepare(State) ->
+    {ok, _} = xref:start(xref),
+    ok = xref:set_library_path(xref, code_path(State)),
+
+    xref:set_default(xref, [{warnings,
+                             rebar_state:get(State, xref_warnings, false)},
+                            {verbose, rebar_log:is_verbose(State)}]),
+
+    {ok, _} = xref:add_directory(xref, "ebin"),
+
+    %% Save the code path prior to doing any further code path
+    %% manipulation
+    OriginalPath = code:get_path(),
+    true = code:add_path(rebar_dir:ebin_dir()),
+
+    %% Get list of xref checks we want to run
+    ConfXrefChecks = rebar_state:get(State, xref_checks,
+                                     [exports_not_used,
+                                      undefined_function_calls]),
+
+    XrefChecks = sets:to_list(sets:intersection(
+                                sets:from_list(?SUPPORTED_XREFS),
+                                sets:from_list(ConfXrefChecks))),
+    {OriginalPath, XrefChecks}.
+
+cleanup(Path) ->
+    %% Restore the code path using the provided path
+    true = rebar_utils:cleanup_code_path(Path),
+
+    %% Stop xref
+    stopped = xref:stop(xref),
+    ok.
+
+xref_checks(XrefChecks) ->
+    lists:foldl(fun run_xref_check/2, [], XrefChecks).
+
+run_xref_check(XrefCheck, Acc) ->
+    {ok, Results} = xref:analyze(xref, XrefCheck),
+    case filter_xref_results(XrefCheck, Results) of
+        [] ->
+            Acc;
+        FilterResult ->
+            [{XrefCheck, FilterResult} | Acc]
+    end.
+
+check_query({Query, Value}, Acc) ->
+    {ok, Answer} = xref:q(xref, Query),
+    case Answer =:= Value of
+        false ->
+            [{Query, Value, Answer} | Acc];
+        _     ->
+            Acc
+    end.
+
+code_path(State) ->
+    [P || P <- code:get_path() ++
+              rebar_state:get(State, xref_extra_paths, []),
+          filelib:is_dir(P)].
+
+%% Ignore behaviour functions, and explicitly marked functions
+%%
+%% Functions can be ignored by using
+%% -ignore_xref([{F, A}, {M, F, A}...]).
+get_xref_ignorelist(Mod, XrefCheck) ->
+    %% Get ignore_xref attribute and combine them in one list
+    Attributes =
+        try
+            Mod:module_info(attributes)
+        catch
+            _Class:_Error -> []
+        end,
+
+    IgnoreXref = keyall(ignore_xref, Attributes),
+
+    BehaviourCallbacks = get_behaviour_callbacks(XrefCheck, Attributes),
+
+    %% And create a flat {M,F,A} list
+    lists:foldl(
+      fun({F, A}, Acc) -> [{Mod,F,A} | Acc];
+         ({M, F, A}, Acc) -> [{M,F,A} | Acc]
+      end, [], lists:flatten([IgnoreXref, BehaviourCallbacks])).
+
+keyall(Key, List) ->
+    lists:flatmap(fun({K, L}) when Key =:= K -> L; (_) -> [] end, List).
+
+get_behaviour_callbacks(exports_not_used, Attributes) ->
+    [B:behaviour_info(callbacks) || B <- keyall(behaviour, Attributes)];
+get_behaviour_callbacks(_XrefCheck, _Attributes) ->
+    [].
+
+parse_xref_result({_, MFAt}) -> MFAt;
+parse_xref_result(MFAt) -> MFAt.
+
+filter_xref_results(XrefCheck, XrefResults) ->
+    SearchModules = lists:usort(
+                      lists:map(
+                        fun({Mt,_Ft,_At}) -> Mt;
+                           ({{Ms,_Fs,_As},{_Mt,_Ft,_At}}) -> Ms;
+                           (_) -> undefined
+                        end, XrefResults)),
+
+    Ignores = lists:flatmap(fun(Module) ->
+                                    get_xref_ignorelist(Module, XrefCheck)
+                            end, SearchModules),
+
+    [Result || Result <- XrefResults,
+               not lists:member(parse_xref_result(Result), Ignores)].
+
+display_results(XrefResults, QueryResults) ->
+    [lists:map(fun display_xref_results_for_type/1, XrefResults),
+     lists:map(fun display_query_result/1, QueryResults)].
+
+display_query_result({Query, Answer, Value}) ->
+    io_lib:format("Query ~s~n answer ~p~n did not match ~p~n",
+                  [Query, Answer, Value]).
+
+display_xref_results_for_type({Type, XrefResults}) ->
+    lists:map(display_xref_result_fun(Type), XrefResults).
+
+display_xref_result_fun(Type) ->
+    fun(XrefResult) ->
+            {Source, SMFA, TMFA} =
+                case XrefResult of
+                    {MFASource, MFATarget} ->
+                        {format_mfa_source(MFASource),
+                         format_mfa(MFASource),
+                         format_mfa(MFATarget)};
+                    MFATarget ->
+                        {format_mfa_source(MFATarget),
+                         format_mfa(MFATarget),
+                         undefined}
+                end,
+            case Type of
+                undefined_function_calls ->
+                    io_lib:format("~sWarning: ~s calls undefined function ~s (Xref)\n",
+                                  [Source, SMFA, TMFA]);
+                undefined_functions ->
+                    io_lib:format("~sWarning: ~s is undefined function (Xref)\n",
+                                  [Source, SMFA]);
+                locals_not_used ->
+                    io_lib:format("~sWarning: ~s is unused local function (Xref)\n",
+                                  [Source, SMFA]);
+                exports_not_used ->
+                    io_lib:format("~sWarning: ~s is unused export (Xref)\n",
+                                  [Source, SMFA]);
+                deprecated_function_calls ->
+                    io_lib:format("~sWarning: ~s calls deprecated function ~s (Xref)\n",
+                                  [Source, SMFA, TMFA]);
+                deprecated_functions ->
+                    io_lib:format("~sWarning: ~s is deprecated function (Xref)\n",
+                                  [Source, SMFA]);
+                Other ->
+                    io_lib:format("~sWarning: ~s - ~s xref check: ~s (Xref)\n",
+                                  [Source, SMFA, TMFA, Other])
+            end
+    end.
+
+format_mfa({M, F, A}) ->
+    ?FMT("~s:~s/~w", [M, F, A]).
+
+format_mfa_source(MFA) ->
+    case find_mfa_source(MFA) of
+        {module_not_found, function_not_found} -> "";
+        {Source, function_not_found} -> ?FMT("~s: ", [Source]);
+        {Source, Line} -> ?FMT("~s:~w: ", [Source, Line])
+    end.
+
+%%
+%% Extract an element from a tuple, or undefined if N > tuple size
+%%
+safe_element(N, Tuple) ->
+    case catch(element(N, Tuple)) of
+        {'EXIT', {badarg, _}} ->
+            undefined;
+        Value ->
+            Value
+    end.
+
+%%
+%% Given a MFA, find the file and LOC where it's defined. Note that
+%% xref doesn't work if there is no abstract_code, so we can avoid
+%% being too paranoid here.
+%%
+find_mfa_source({M, F, A}) ->
+    case code:get_object_code(M) of
+        error -> {module_not_found, function_not_found};
+        {M, Bin, _} -> find_function_source(M,F,A,Bin)
+    end.
+
+find_function_source(M, F, A, Bin) ->
+    AbstractCode = beam_lib:chunks(Bin, [abstract_code]),
+    {ok, {M, [{abstract_code, {raw_abstract_v1, Code}}]}} = AbstractCode,
+    %% Extract the original source filename from the abstract code
+    [{attribute, 1, file, {Source, _}} | _] = Code,
+    %% Extract the line number for a given function def
+    Fn = [E || E <- Code,
+               safe_element(1, E) == function,
+               safe_element(3, E) == F,
+               safe_element(4, E) == A],
+    case Fn of
+        [{function, Line, F, _, _}] -> {Source, Line};
+        %% do not crash if functions are exported, even though they
+        %% are not in the source.
+        %% parameterized modules add new/1 and instance/1 for example.
+        [] -> {Source, function_not_found}
+    end.

test/rebar_xref_SUITE.erl

@@ -0,0 +1,190 @@
+%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*-
+%% ex: ts=4 sw=4 et
+-module(rebar_xref_SUITE).
+
+-export([suite/0,
+         init_per_suite/1,
+         end_per_suite/1,
+         init_per_testcase/2,
+         end_per_testcase/2,
+         all/0,
+         xref_test/1,
+         xref_ignore_test/1]).
+
+-include_lib("common_test/include/ct.hrl").
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("kernel/include/file.hrl").
+
+%% ===================================================================
+%% common_test callbacks
+%% ===================================================================
+
+suite() ->
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_testcase(Case, Config) ->
+    UpdConfig = rebar_test_utils:init_rebar_state(Config),
+    AppDir = ?config(apps, UpdConfig),
+    {ok, OrigDir} = file:get_cwd(),
+    file:set_cwd(AppDir),
+    Name = rebar_test_utils:create_random_name("xrefapp_"),
+    Vsn = rebar_test_utils:create_random_vsn(),
+    rebar_test_utils:create_empty_app(AppDir, Name, Vsn, [kernel, stdlib]),
+    AppModules = [behaviour1, behaviour2, mymod, othermod],
+    [write_src_file(AppDir, Name, Module, ignore_xref(Case)) || Module <- AppModules],
+    RebarConfig = [{erl_opts, [debug_info]},
+                   {xref_checks, [deprecated_function_calls,deprecated_functions,
+                                  undefined_function_calls,undefined_functions,
+                                  exports_not_used,locals_not_used]}],
+    [{app_name, Name},
+     {rebar_config, RebarConfig},
+     {orig_dir, OrigDir} | UpdConfig].
+
+end_per_testcase(_, Config) ->
+    ?debugMsg("End test case cleanup"),
+    AppDir = ?config(apps, Config),
+    OrigDir = ?config(orig_dir, Config),
+    %% Code path cleanup because we set the CWD to the `AppDir' prior
+    %% to launching rebar and these paths make it into the code path
+    %% before the xref module executes so they don't get cleaned up
+    %% automatically after the xref run. Only have to do this because
+    %% we are about to remove the directory and there may be
+    %% subsequent test cases that error out when the code path tries
+    %% to include one of these soon-to-be nonexistent directories.
+    true = code:del_path(AppDir ++ "/."),
+    true = code:del_path(rebar_dir:ebin_dir()),
+    file:set_cwd(OrigDir),
+    ec_file:remove(AppDir, [recursive]),
+    ok.
+
+all() ->
+    [xref_test, xref_ignore_test].
+
+%% ===================================================================
+%% Test cases
+%% ===================================================================
+
+xref_test(Config) ->
+    AppDir = ?config(apps, Config),
+    State = ?config(state, Config),
+    Name = ?config(app_name, Config),
+    RebarConfig = ?config(rebar_config, Config),
+    Result = rebar3:run(rebar_state:new(State, RebarConfig, AppDir), ["xref"]),
+    verify_results(xref_test, Name, Result).
+
+xref_ignore_test(Config) ->
+    AppDir = ?config(apps, Config),
+    State = ?config(state, Config),
+    Name = ?config(app_name, Config),
+    RebarConfig = ?config(rebar_config, Config),
+    Result = rebar3:run(rebar_state:new(State, RebarConfig, AppDir), ["xref"]),
+    verify_results(xref_ignore_test, Name, Result).
+
+%% ===================================================================
+%% Helper functions
+%% ===================================================================
+
+ignore_xref(xref_ignore_test) ->
+    true;
+ignore_xref(_) ->
+    false.
+
+verify_results(TestCase, AppName, Results) ->
+    {error, {rebar_prv_xref,
+             {xref_issues, XrefResults, QueryResults}}} = Results,
+    verify_test_results(TestCase, AppName, XrefResults, QueryResults).
+
+verify_test_results(xref_test, AppName, XrefResults, _QueryResults) ->
+    AppModules = ["behaviour1", "behaviour2", "mymod", "othermod", "somemod"],
+    [Behaviour1Mod, Behaviour2Mod, MyMod, OtherMod, SomeMod] =
+        [list_to_atom(AppName ++ "_" ++ Mod) || Mod <- AppModules],
+    UndefFuns = proplists:get_value(undefined_functions, XrefResults),
+    UndefFunCalls = proplists:get_value(undefined_function_calls, XrefResults),
+    LocalsNotUsed = proplists:get_value(locals_not_used, XrefResults),
+    ExportsNotUsed = proplists:get_value(exports_not_used, XrefResults),
+    DeprecatedFuns = proplists:get_value(deprecated_functions, XrefResults),
+    DeprecatedFunCalls = proplists:get_value(deprecated_function_calls, XrefResults),
+    ?assert(lists:member({SomeMod, notavailable, 1}, UndefFuns)),
+    ?assert(lists:member({{OtherMod, somefunc, 0}, {SomeMod, notavailable, 1}},
+                         UndefFunCalls)),
+    ?assert(lists:member({MyMod, fdeprecated, 0}, DeprecatedFuns)),
+    ?assert(lists:member({{OtherMod, somefunc, 0}, {MyMod, fdeprecated, 0}},
+                         DeprecatedFunCalls)),
+    ?assert(lists:member({MyMod, localfunc2, 0}, LocalsNotUsed)),
+    ?assert(lists:member({Behaviour1Mod, behaviour_info, 1}, ExportsNotUsed)),
+    ?assert(lists:member({Behaviour2Mod, behaviour_info, 1}, ExportsNotUsed)),
+    ?assert(lists:member({MyMod, other2, 1}, ExportsNotUsed)),
+    ?assert(lists:member({OtherMod, somefunc, 0}, ExportsNotUsed)),
+    ?assertNot(lists:member({MyMod, bh1_a, 1}, ExportsNotUsed)),
+    ?assertNot(lists:member({MyMod, bh1_b, 1}, ExportsNotUsed)),
+    ?assertNot(lists:member({MyMod, bh2_a, 1}, ExportsNotUsed)),
+    ?assertNot(lists:member({MyMod, bh2_b, 1}, ExportsNotUsed)),
+    ok;
+verify_test_results(xref_ignore_test, AppName, XrefResults, _QueryResults) ->
+    AppModules = ["behaviour1", "behaviour2", "mymod", "othermod", "somemod"],
+    [Behaviour1Mod, Behaviour2Mod, MyMod, OtherMod, SomeMod] =
+        [list_to_atom(AppName ++ "_" ++ Mod) || Mod <- AppModules],
+    UndefFuns = proplists:get_value(undefined_functions, XrefResults),
+    ?assertNot(lists:keymember(undefined_function_calls, 1, XrefResults)),
+    ?assertNot(lists:keymember(locals_not_used, 1, XrefResults)),
+    ?assertNot(lists:keymember(exports_not_used, 1, XrefResults)),
+    ?assertNot(lists:keymember(deprecated_functions, 1, XrefResults)),
+    ?assertNot(lists:keymember(deprecated_function_calls, 1, XrefResults)),
+    ?assert(lists:member({SomeMod, notavailable, 1}, UndefFuns)),
+    ok.
+
+write_src_file(Dir, AppName, Module, IgnoreXref) ->
+    Erl = filename:join([Dir, "src", module_name(AppName, Module)]),
+    ok = filelib:ensure_dir(Erl),
+    ok = ec_file:write(Erl, get_module_body(Module, AppName, IgnoreXref)).
+
+module_name(AppName, Module) ->
+    lists:flatten([AppName, "_", atom_to_list(Module), ".erl"]).
+
+get_module_body(behaviour1, AppName, IgnoreXref) ->
+    ["-module(", AppName, "_behaviour1).\n",
+     "-export([behaviour_info/1]).\n",
+     ["-ignore_xref({behaviour_info,1}).\n" || X <- [IgnoreXref], X =:= true],
+     "behaviour_info(callbacks) -> [{bh1_a,1},{bh1_b,1}];\n",
+     "behaviour_info(_Other) -> undefined.\n"];
+get_module_body(behaviour2, AppName, IgnoreXref) ->
+    ["-module(", AppName, "_behaviour2).\n",
+     "-export([behaviour_info/1]).\n",
+     ["-ignore_xref({behaviour_info,1}).\n" || X <- [IgnoreXref], X =:= true],
+     "behaviour_info(callbacks) -> [{bh2_a,1},{bh2_b,1}];\n",
+     "behaviour_info(_Other) -> undefined.\n"];
+get_module_body(mymod, AppName, IgnoreXref) ->
+    ["-module(", AppName, "_mymod).\n",
+     "-export([bh1_a/1,bh1_b/1,bh2_a/1,bh2_b/1,"
+     "other1/1,other2/1,fdeprecated/0]).\n",
+     ["-ignore_xref([{other2,1},{localfunc2,0},{fdeprecated,0}]).\n"
+      || X <- [IgnoreXref], X =:= true],
+     "-behaviour(", AppName, "_behaviour1).\n",     % 2 behaviours
+     "-behaviour(", AppName, "_behaviour2).\n",
+     "-deprecated({fdeprecated,0}).\n",      % deprecated function
+     "bh1_a(A) -> localfunc1(bh1_a, A).\n", % behaviour functions
+     "bh1_b(A) -> localfunc1(bh1_b, A).\n",
+     "bh2_a(A) -> localfunc1(bh2_a, A).\n",
+     "bh2_b(A) -> localfunc1(bh2_b, A).\n",
+     "other1(A) -> localfunc1(other1, A).\n", % regular exported functions
+     "other2(A) -> localfunc1(other2, A).\n",
+     "localfunc1(A, B) -> {A, B}.\n",       % used local
+     "localfunc2() -> ok.\n",               % unused local
+     "fdeprecated() -> ok.\n"              % deprecated function
+    ];
+get_module_body(othermod, AppName, IgnoreXref) ->
+    ["-module(", AppName, "_othermod).\n",
+     "-export([somefunc/0]).\n",
+     [["-ignore_xref([{", AppName, "_somemod,notavailable,1},{somefunc,0}]).\n",
+       "-ignore_xref({", AppName, "_mymod,fdeprecated,0}).\n"]
+      || X <- [IgnoreXref], X =:= true],
+     "somefunc() ->\n",
+     "   ", AppName, "_mymod:other1(arg),\n",
+     "   ", AppName, "_somemod:notavailable(arg),\n",
+     "   ", AppName, "_mymod:fdeprecated().\n"].