Merge pull request #127 from kellymclaughlin/xref-provider

Add xref provider for cross reference analysis

README.md

@@ -43,6 +43,7 @@ limit scope.
 | update     | Update package index |
 | upgrade    | Fetch latest version of dep |
 | version    | Print current version of Erlang/OTP and rebar |
+| xref       | Run cross reference analysis on the project |
 
 ### Commands still to do
 

THANKS

@@ -128,4 +128,5 @@ Alexander Verbitsky
 Andras Horvath
 Drew Varner
 Omar Yasin
-Tristan Sloughter
+Tristan Sloughter
+Kelly McLaughlin

src/rebar.app.src

@@ -43,6 +43,7 @@
                      rebar_prv_release,
                      rebar_prv_version,
                      rebar_prv_common_test,
+                     rebar_prv_xref,
                      rebar_prv_help]}
         ]}
  ]}.

src/rebar_prv_xref.erl

@@ -0,0 +1,296 @@
+%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*-
+%% ex: ts=4 sw=4 et
+
+-module(rebar_prv_xref).
+
+-behaviour(provider).
+
+-export([init/1,
+         do/1,
+         format_error/1]).
+
+-include("rebar.hrl").
+
+-define(PROVIDER, xref).
+-define(DEPS, [compile]).
+-define(SUPPORTED_XREFS, [undefined_function_calls, undefined_functions,
+                          locals_not_used, exports_not_used,
+                          deprecated_function_calls, deprecated_functions]).
+
+%% ===================================================================
+%% Public API
+%% ===================================================================
+
+-spec init(rebar_state:t()) -> {ok, rebar_state:t()}.
+init(State) ->
+    Provider = providers:create([{name, ?PROVIDER},
+                                 {module, ?MODULE},
+                                 {deps, ?DEPS},
+                                 {bare, false},
+                                 {example, "rebar3 xref"},
+                                 {short_desc, short_desc()},
+                                 {desc, desc()}]),
+    State1 = rebar_state:add_provider(State, Provider),
+    {ok, State1}.
+
+-spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}.
+do(State) ->
+    {OriginalPath, XrefChecks} = prepare(State),
+
+    %% Run xref checks
+    ?INFO("Running cross reference analysis...", []),
+    XrefResults = xref_checks(XrefChecks),
+
+    %% Run custom queries
+    QueryChecks = rebar_state:get(State, xref_queries, []),
+    QueryResults = lists:foldl(fun check_query/2, [], QueryChecks),
+
+    ok = cleanup(OriginalPath),
+
+    case XrefResults =:= [] andalso QueryResults =:= [] of
+        true ->
+            {ok, State};
+        false ->
+            {error, {?MODULE, {xref_issues, XrefResults, QueryResults}}}
+    end.
+
+-spec format_error(any()) -> iolist().
+format_error({xref_issues, XrefResults, QueryResults}) ->
+    lists:flatten(display_results(XrefResults, QueryResults));
+format_error(Reason) ->
+    io_lib:format("~p", [Reason]).
+
+%% ===================================================================
+%% Internal functions
+%% ===================================================================
+
+short_desc() ->
+    "Run cross reference analysis".
+
+desc() ->
+    io_lib:format(
+      "~s~n"
+      "~n"
+      "Valid rebar.config options:~n"
+      "  ~p~n"
+      "  ~p~n"
+      "  ~p~n"
+      "  ~p~n",
+      [short_desc(),
+       {xref_warnings, false},
+       {xref_extra_paths,[]},
+       {xref_checks, [undefined_function_calls, undefined_functions,
+                      locals_not_used, exports_not_used,
+                      deprecated_function_calls, deprecated_functions]},
+       {xref_queries,
+        [{"(xc - uc) || (xu - x - b"
+          " - (\"mod\":\".*foo\"/\"4\"))",[]}]}
+      ]).
+
+-spec prepare(rebar_state:t()) -> list(atom()).
+prepare(State) ->
+    {ok, _} = xref:start(xref),
+    ok = xref:set_library_path(xref, code_path(State)),
+
+    xref:set_default(xref, [{warnings,
+                             rebar_state:get(State, xref_warnings, false)},
+                            {verbose, rebar_log:is_verbose(State)}]),
+
+    {ok, _} = xref:add_directory(xref, "ebin"),
+
+    %% Save the code path prior to doing any further code path
+    %% manipulation
+    OriginalPath = code:get_path(),
+    true = code:add_path(rebar_dir:ebin_dir()),
+
+    %% Get list of xref checks we want to run
+    ConfXrefChecks = rebar_state:get(State, xref_checks,
+                                     [exports_not_used,
+                                      undefined_function_calls]),
+
+    XrefChecks = sets:to_list(sets:intersection(
+                                sets:from_list(?SUPPORTED_XREFS),
+                                sets:from_list(ConfXrefChecks))),
+    {OriginalPath, XrefChecks}.
+
+cleanup(Path) ->
+    %% Restore the code path using the provided path
+    true = rebar_utils:cleanup_code_path(Path),
+
+    %% Stop xref
+    stopped = xref:stop(xref),
+    ok.
+
+xref_checks(XrefChecks) ->
+    lists:foldl(fun run_xref_check/2, [], XrefChecks).
+
+run_xref_check(XrefCheck, Acc) ->
+    {ok, Results} = xref:analyze(xref, XrefCheck),
+    case filter_xref_results(XrefCheck, Results) of
+        [] ->
+            Acc;
+        FilterResult ->
+            [{XrefCheck, FilterResult} | Acc]
+    end.
+
+check_query({Query, Value}, Acc) ->
+    {ok, Answer} = xref:q(xref, Query),
+    case Answer =:= Value of
+        false ->
+            [{Query, Value, Answer} | Acc];
+        _     ->
+            Acc
+    end.
+
+code_path(State) ->
+    [P || P <- code:get_path() ++
+              rebar_state:get(State, xref_extra_paths, []),
+          filelib:is_dir(P)].
+
+%% Ignore behaviour functions, and explicitly marked functions
+%%
+%% Functions can be ignored by using
+%% -ignore_xref([{F, A}, {M, F, A}...]).
+get_xref_ignorelist(Mod, XrefCheck) ->
+    %% Get ignore_xref attribute and combine them in one list
+    Attributes =
+        try
+            Mod:module_info(attributes)
+        catch
+            _Class:_Error -> []
+        end,
+
+    IgnoreXref = keyall(ignore_xref, Attributes),
+
+    BehaviourCallbacks = get_behaviour_callbacks(XrefCheck, Attributes),
+
+    %% And create a flat {M,F,A} list
+    lists:foldl(
+      fun({F, A}, Acc) -> [{Mod,F,A} | Acc];
+         ({M, F, A}, Acc) -> [{M,F,A} | Acc]
+      end, [], lists:flatten([IgnoreXref, BehaviourCallbacks])).
+
+keyall(Key, List) ->
+    lists:flatmap(fun({K, L}) when Key =:= K -> L; (_) -> [] end, List).
+
+get_behaviour_callbacks(exports_not_used, Attributes) ->
+    [B:behaviour_info(callbacks) || B <- keyall(behaviour, Attributes)];
+get_behaviour_callbacks(_XrefCheck, _Attributes) ->
+    [].
+
+parse_xref_result({_, MFAt}) -> MFAt;
+parse_xref_result(MFAt) -> MFAt.
+
+filter_xref_results(XrefCheck, XrefResults) ->
+    SearchModules = lists:usort(
+                      lists:map(
+                        fun({Mt,_Ft,_At}) -> Mt;
+                           ({{Ms,_Fs,_As},{_Mt,_Ft,_At}}) -> Ms;
+                           (_) -> undefined
+                        end, XrefResults)),
+
+    Ignores = lists:flatmap(fun(Module) ->
+                                    get_xref_ignorelist(Module, XrefCheck)
+                            end, SearchModules),
+
+    [Result || Result <- XrefResults,
+               not lists:member(parse_xref_result(Result), Ignores)].
+
+display_results(XrefResults, QueryResults) ->
+    [lists:map(fun display_xref_results_for_type/1, XrefResults),
+     lists:map(fun display_query_result/1, QueryResults)].
+
+display_query_result({Query, Answer, Value}) ->
+    io_lib:format("Query ~s~n answer ~p~n did not match ~p~n",
+                  [Query, Answer, Value]).
+
+display_xref_results_for_type({Type, XrefResults}) ->
+    lists:map(display_xref_result_fun(Type), XrefResults).
+
+display_xref_result_fun(Type) ->
+    fun(XrefResult) ->
+            {Source, SMFA, TMFA} =
+                case XrefResult of
+                    {MFASource, MFATarget} ->
+                        {format_mfa_source(MFASource),
+                         format_mfa(MFASource),
+                         format_mfa(MFATarget)};
+                    MFATarget ->
+                        {format_mfa_source(MFATarget),
+                         format_mfa(MFATarget),
+                         undefined}
+                end,
+            case Type of
+                undefined_function_calls ->
+                    io_lib:format("~sWarning: ~s calls undefined function ~s (Xref)\n",
+                                  [Source, SMFA, TMFA]);
+                undefined_functions ->
+                    io_lib:format("~sWarning: ~s is undefined function (Xref)\n",
+                                  [Source, SMFA]);
+                locals_not_used ->
+                    io_lib:format("~sWarning: ~s is unused local function (Xref)\n",
+                                  [Source, SMFA]);
+                exports_not_used ->
+                    io_lib:format("~sWarning: ~s is unused export (Xref)\n",
+                                  [Source, SMFA]);
+                deprecated_function_calls ->
+                    io_lib:format("~sWarning: ~s calls deprecated function ~s (Xref)\n",
+                                  [Source, SMFA, TMFA]);
+                deprecated_functions ->
+                    io_lib:format("~sWarning: ~s is deprecated function (Xref)\n",
+                                  [Source, SMFA]);
+                Other ->
+                    io_lib:format("~sWarning: ~s - ~s xref check: ~s (Xref)\n",
+                                  [Source, SMFA, TMFA, Other])
+            end
+    end.
+
+format_mfa({M, F, A}) ->
+    ?FMT("~s:~s/~w", [M, F, A]).
+
+format_mfa_source(MFA) ->
+    case find_mfa_source(MFA) of
+        {module_not_found, function_not_found} -> "";
+        {Source, function_not_found} -> ?FMT("~s: ", [Source]);
+        {Source, Line} -> ?FMT("~s:~w: ", [Source, Line])
+    end.
+
+%%
+%% Extract an element from a tuple, or undefined if N > tuple size
+%%
+safe_element(N, Tuple) ->
+    case catch(element(N, Tuple)) of
+        {'EXIT', {badarg, _}} ->
+            undefined;
+        Value ->
+            Value
+    end.
+
+%%
+%% Given a MFA, find the file and LOC where it's defined. Note that
+%% xref doesn't work if there is no abstract_code, so we can avoid
+%% being too paranoid here.
+%%
+find_mfa_source({M, F, A}) ->
+    case code:get_object_code(M) of
+        error -> {module_not_found, function_not_found};
+        {M, Bin, _} -> find_function_source(M,F,A,Bin)
+    end.
+
+find_function_source(M, F, A, Bin) ->
+    AbstractCode = beam_lib:chunks(Bin, [abstract_code]),
+    {ok, {M, [{abstract_code, {raw_abstract_v1, Code}}]}} = AbstractCode,
+    %% Extract the original source filename from the abstract code
+    [{attribute, 1, file, {Source, _}} | _] = Code,
+    %% Extract the line number for a given function def
+    Fn = [E || E <- Code,
+               safe_element(1, E) == function,
+               safe_element(3, E) == F,
+               safe_element(4, E) == A],
+    case Fn of
+        [{function, Line, F, _, _}] -> {Source, Line};
+        %% do not crash if functions are exported, even though they
+        %% are not in the source.
+        %% parameterized modules add new/1 and instance/1 for example.
+        [] -> {Source, function_not_found}
+    end.

test/rebar_xref_SUITE.erl

@@ -0,0 +1,190 @@
+%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*-
+%% ex: ts=4 sw=4 et
+-module(rebar_xref_SUITE).
+
+-export([suite/0,
+         init_per_suite/1,
+         end_per_suite/1,
+         init_per_testcase/2,
+         end_per_testcase/2,
+         all/0,
+         xref_test/1,
+         xref_ignore_test/1]).
+
+-include_lib("common_test/include/ct.hrl").
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("kernel/include/file.hrl").
+
+%% ===================================================================
+%% common_test callbacks
+%% ===================================================================
+
+suite() ->
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_testcase(Case, Config) ->
+    UpdConfig = rebar_test_utils:init_rebar_state(Config),
+    AppDir = ?config(apps, UpdConfig),
+    {ok, OrigDir} = file:get_cwd(),
+    file:set_cwd(AppDir),
+    Name = rebar_test_utils:create_random_name("xrefapp_"),
+    Vsn = rebar_test_utils:create_random_vsn(),
+    rebar_test_utils:create_empty_app(AppDir, Name, Vsn, [kernel, stdlib]),
+    AppModules = [behaviour1, behaviour2, mymod, othermod],
+    [write_src_file(AppDir, Name, Module, ignore_xref(Case)) || Module <- AppModules],
+    RebarConfig = [{erl_opts, [debug_info]},
+                   {xref_checks, [deprecated_function_calls,deprecated_functions,
+                                  undefined_function_calls,undefined_functions,
+                                  exports_not_used,locals_not_used]}],
+    [{app_name, Name},
+     {rebar_config, RebarConfig},
+     {orig_dir, OrigDir} | UpdConfig].
+
+end_per_testcase(_, Config) ->
+    ?debugMsg("End test case cleanup"),
+    AppDir = ?config(apps, Config),
+    OrigDir = ?config(orig_dir, Config),
+    %% Code path cleanup because we set the CWD to the `AppDir' prior
+    %% to launching rebar and these paths make it into the code path
+    %% before the xref module executes so they don't get cleaned up
+    %% automatically after the xref run. Only have to do this because
+    %% we are about to remove the directory and there may be
+    %% subsequent test cases that error out when the code path tries
+    %% to include one of these soon-to-be nonexistent directories.
+    true = code:del_path(AppDir ++ "/."),
+    true = code:del_path(rebar_dir:ebin_dir()),
+    file:set_cwd(OrigDir),
+    ec_file:remove(AppDir, [recursive]),
+    ok.
+
+all() ->
+    [xref_test, xref_ignore_test].
+
+%% ===================================================================
+%% Test cases
+%% ===================================================================
+
+xref_test(Config) ->
+    AppDir = ?config(apps, Config),
+    State = ?config(state, Config),
+    Name = ?config(app_name, Config),
+    RebarConfig = ?config(rebar_config, Config),
+    Result = rebar3:run(rebar_state:new(State, RebarConfig, AppDir), ["xref"]),
+    verify_results(xref_test, Name, Result).
+
+xref_ignore_test(Config) ->
+    AppDir = ?config(apps, Config),
+    State = ?config(state, Config),
+    Name = ?config(app_name, Config),
+    RebarConfig = ?config(rebar_config, Config),
+    Result = rebar3:run(rebar_state:new(State, RebarConfig, AppDir), ["xref"]),
+    verify_results(xref_ignore_test, Name, Result).
+
+%% ===================================================================
+%% Helper functions
+%% ===================================================================
+
+ignore_xref(xref_ignore_test) ->
+    true;
+ignore_xref(_) ->
+    false.
+
+verify_results(TestCase, AppName, Results) ->
+    {error, {rebar_prv_xref,
+             {xref_issues, XrefResults, QueryResults}}} = Results,
+    verify_test_results(TestCase, AppName, XrefResults, QueryResults).
+
+verify_test_results(xref_test, AppName, XrefResults, _QueryResults) ->
+    AppModules = ["behaviour1", "behaviour2", "mymod", "othermod", "somemod"],
+    [Behaviour1Mod, Behaviour2Mod, MyMod, OtherMod, SomeMod] =
+        [list_to_atom(AppName ++ "_" ++ Mod) || Mod <- AppModules],
+    UndefFuns = proplists:get_value(undefined_functions, XrefResults),
+    UndefFunCalls = proplists:get_value(undefined_function_calls, XrefResults),
+    LocalsNotUsed = proplists:get_value(locals_not_used, XrefResults),
+    ExportsNotUsed = proplists:get_value(exports_not_used, XrefResults),
+    DeprecatedFuns = proplists:get_value(deprecated_functions, XrefResults),
+    DeprecatedFunCalls = proplists:get_value(deprecated_function_calls, XrefResults),
+    ?assert(lists:member({SomeMod, notavailable, 1}, UndefFuns)),
+    ?assert(lists:member({{OtherMod, somefunc, 0}, {SomeMod, notavailable, 1}},
+                         UndefFunCalls)),
+    ?assert(lists:member({MyMod, fdeprecated, 0}, DeprecatedFuns)),
+    ?assert(lists:member({{OtherMod, somefunc, 0}, {MyMod, fdeprecated, 0}},
+                         DeprecatedFunCalls)),
+    ?assert(lists:member({MyMod, localfunc2, 0}, LocalsNotUsed)),
+    ?assert(lists:member({Behaviour1Mod, behaviour_info, 1}, ExportsNotUsed)),
+    ?assert(lists:member({Behaviour2Mod, behaviour_info, 1}, ExportsNotUsed)),
+    ?assert(lists:member({MyMod, other2, 1}, ExportsNotUsed)),
+    ?assert(lists:member({OtherMod, somefunc, 0}, ExportsNotUsed)),
+    ?assertNot(lists:member({MyMod, bh1_a, 1}, ExportsNotUsed)),
+    ?assertNot(lists:member({MyMod, bh1_b, 1}, ExportsNotUsed)),
+    ?assertNot(lists:member({MyMod, bh2_a, 1}, ExportsNotUsed)),
+    ?assertNot(lists:member({MyMod, bh2_b, 1}, ExportsNotUsed)),
+    ok;
+verify_test_results(xref_ignore_test, AppName, XrefResults, _QueryResults) ->
+    AppModules = ["behaviour1", "behaviour2", "mymod", "othermod", "somemod"],
+    [Behaviour1Mod, Behaviour2Mod, MyMod, OtherMod, SomeMod] =
+        [list_to_atom(AppName ++ "_" ++ Mod) || Mod <- AppModules],
+    UndefFuns = proplists:get_value(undefined_functions, XrefResults),
+    ?assertNot(lists:keymember(undefined_function_calls, 1, XrefResults)),
+    ?assertNot(lists:keymember(locals_not_used, 1, XrefResults)),
+    ?assertNot(lists:keymember(exports_not_used, 1, XrefResults)),
+    ?assertNot(lists:keymember(deprecated_functions, 1, XrefResults)),
+    ?assertNot(lists:keymember(deprecated_function_calls, 1, XrefResults)),
+    ?assert(lists:member({SomeMod, notavailable, 1}, UndefFuns)),
+    ok.
+
+write_src_file(Dir, AppName, Module, IgnoreXref) ->
+    Erl = filename:join([Dir, "src", module_name(AppName, Module)]),
+    ok = filelib:ensure_dir(Erl),
+    ok = ec_file:write(Erl, get_module_body(Module, AppName, IgnoreXref)).
+
+module_name(AppName, Module) ->
+    lists:flatten([AppName, "_", atom_to_list(Module), ".erl"]).
+
+get_module_body(behaviour1, AppName, IgnoreXref) ->
+    ["-module(", AppName, "_behaviour1).\n",
+     "-export([behaviour_info/1]).\n",
+     ["-ignore_xref({behaviour_info,1}).\n" || X <- [IgnoreXref], X =:= true],
+     "behaviour_info(callbacks) -> [{bh1_a,1},{bh1_b,1}];\n",
+     "behaviour_info(_Other) -> undefined.\n"];
+get_module_body(behaviour2, AppName, IgnoreXref) ->
+    ["-module(", AppName, "_behaviour2).\n",
+     "-export([behaviour_info/1]).\n",
+     ["-ignore_xref({behaviour_info,1}).\n" || X <- [IgnoreXref], X =:= true],
+     "behaviour_info(callbacks) -> [{bh2_a,1},{bh2_b,1}];\n",
+     "behaviour_info(_Other) -> undefined.\n"];
+get_module_body(mymod, AppName, IgnoreXref) ->
+    ["-module(", AppName, "_mymod).\n",
+     "-export([bh1_a/1,bh1_b/1,bh2_a/1,bh2_b/1,"
+     "other1/1,other2/1,fdeprecated/0]).\n",
+     ["-ignore_xref([{other2,1},{localfunc2,0},{fdeprecated,0}]).\n"
+      || X <- [IgnoreXref], X =:= true],
+     "-behaviour(", AppName, "_behaviour1).\n",     % 2 behaviours
+     "-behaviour(", AppName, "_behaviour2).\n",
+     "-deprecated({fdeprecated,0}).\n",      % deprecated function
+     "bh1_a(A) -> localfunc1(bh1_a, A).\n", % behaviour functions
+     "bh1_b(A) -> localfunc1(bh1_b, A).\n",
+     "bh2_a(A) -> localfunc1(bh2_a, A).\n",
+     "bh2_b(A) -> localfunc1(bh2_b, A).\n",
+     "other1(A) -> localfunc1(other1, A).\n", % regular exported functions
+     "other2(A) -> localfunc1(other2, A).\n",
+     "localfunc1(A, B) -> {A, B}.\n",       % used local
+     "localfunc2() -> ok.\n",               % unused local
+     "fdeprecated() -> ok.\n"              % deprecated function
+    ];
+get_module_body(othermod, AppName, IgnoreXref) ->
+    ["-module(", AppName, "_othermod).\n",
+     "-export([somefunc/0]).\n",
+     [["-ignore_xref([{", AppName, "_somemod,notavailable,1},{somefunc,0}]).\n",
+       "-ignore_xref({", AppName, "_mymod,fdeprecated,0}).\n"]
+      || X <- [IgnoreXref], X =:= true],
+     "somefunc() ->\n",
+     "   ", AppName, "_mymod:other1(arg),\n",
+     "   ", AppName, "_somemod:notavailable(arg),\n",
+     "   ", AppName, "_mymod:fdeprecated().\n"].