jiffy/test/etap.erl

%% Copyright (c) 2008-2009 Nick Gerakines <nick@gerakines.net>
%%
%% Permission is hereby granted, free of charge, to any person
%% obtaining a copy of this software and associated documentation
%% files (the "Software"), to deal in the Software without
%% restriction, including without limitation the rights to use,
%% copy, modify, merge, publish, distribute, sublicense, and/or sell
%% copies of the Software, and to permit persons to whom the
%% Software is furnished to do so, subject to the following
%% conditions:
%%
%% The above copyright notice and this permission notice shall be
%% included in all copies or substantial portions of the Software.
%%
%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
%% EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
%% OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
%% NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
%% HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
%% WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
%% FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
%% OTHER DEALINGS IN THE SOFTWARE.
%%
%% @author Nick Gerakines <nick@gerakines.net> [http://socklabs.com/]
%% @author Jeremy Wall <jeremy@marzhillstudios.com>
%% @version 0.3.4
%% @copyright 2007-2008 Jeremy Wall, 2008-2009 Nick Gerakines
%% @reference http://testanything.org/wiki/index.php/Main_Page
%% @reference http://en.wikipedia.org/wiki/Test_Anything_Protocol
%% @todo Finish implementing the skip directive.
%% @todo Document the messages handled by this receive loop.
%% @todo Explain in documentation why we use a process to handle test input.
%% @doc etap is a TAP testing module for Erlang components and applications.
%% This module allows developers to test their software using the TAP method.
%%
%% <blockquote cite="http://en.wikipedia.org/wiki/Test_Anything_Protocol"><p>
%% TAP, the Test Anything Protocol, is a simple text-based interface between
%% testing modules in a test harness. TAP started life as part of the test
%% harness for Perl but now has implementations in C/C++, Python, PHP, Perl
%% and probably others by the time you read this.
%% </p></blockquote>
%%
%% The testing process begins by defining a plan using etap:plan/1, running
%% a number of etap tests and then calling eta:end_tests/0. Please refer to
%% the Erlang modules in the t directory of this project for example tests.
-module(etap).
-vsn("0.3.4").

-export([
    ensure_test_server/0,
    start_etap_server/0,
    test_server/1,
    msg/1, msg/2,
    diag/1, diag/2,
    expectation_mismatch_message/3,
    plan/1,
    end_tests/0,
    not_ok/2, ok/2, is_ok/2, is/3, isnt/3, any/3, none/3,
    fun_is/3, expect_fun/3, expect_fun/4,
    is_greater/3,
    skip/1, skip/2,
    datetime/1,
    skip/3,
    bail/0, bail/1,
    test_state/0, failure_count/0
]).

-export([
    contains_ok/3,
    is_before/4
]).

-export([
    is_pid/2,
    is_alive/2,
    is_mfa/3
]).

-export([
    loaded_ok/2,
    can_ok/2, can_ok/3,
    has_attrib/2, is_attrib/3,
    is_behaviour/2
]).

-export([
    dies_ok/2,
    lives_ok/2,
    throws_ok/3
]).


-record(test_state, {
    planned = 0,
    count = 0,
    pass = 0,
    fail = 0,
    skip = 0,
    skip_reason = ""
}).

%% @spec plan(N) -> Result
%%       N = unknown | skip | {skip, string()} | integer()
%%       Result = ok
%% @doc Create a test plan and boot strap the test server.
plan(unknown) ->
    ensure_test_server(),
    etap_server ! {self(), plan, unknown},
    ok;
plan(skip) ->
    io:format("1..0 # skip~n");
plan({skip, Reason}) ->
    io:format("1..0 # skip ~s~n", [Reason]);
plan(N) when is_integer(N), N > 0 ->
    ensure_test_server(),
    etap_server ! {self(), plan, N},
    ok.

%% @spec end_tests() -> ok
%% @doc End the current test plan and output test results.
%% @todo This should probably be done in the test_server process.
end_tests() ->
    case whereis(etap_server) of
        undefined -> self() ! true;
        _ -> etap_server ! {self(), state}
    end,
    State = receive X -> X end,
    if
        State#test_state.planned == -1 ->
            io:format("1..~p~n", [State#test_state.count]);
        true ->
            ok
    end,
    case whereis(etap_server) of
        undefined -> ok;
        _ -> etap_server ! done, ok
    end.

bail() ->
    bail("").

bail(Reason) ->
    etap_server ! {self(), diag, "Bail out! " ++ Reason},
    etap_server ! done, ok,
    ok.

%% @spec test_state() -> Return
%%       Return = test_state_record() | {error, string()}
%% @doc Return the current test state
test_state() ->
    etap_server ! {self(), state},
    receive
	X when is_record(X, test_state) -> X
    after
	1000 -> {error, "Timed out waiting for etap server reply.~n"}
    end.

%% @spec failure_count() -> Return
%%       Return = integer() | {error, string()}
%% @doc Return the current failure count
failure_count() ->
    case test_state() of
        #test_state{fail=FailureCount} -> FailureCount;
        X -> X
    end.

%% @spec msg(S) -> ok
%%       S = string()
%% @doc Print a message in the test output.
msg(S) -> etap_server ! {self(), diag, S}, ok.

%% @spec msg(Format, Data) -> ok
%%      Format = atom() | string() | binary()
%%      Data = [term()]
%%      UnicodeList = [Unicode]
%%      Unicode = int()
%% @doc Print a message in the test output.
%% Function arguments are passed through io_lib:format/2.
msg(Format, Data) -> msg(io_lib:format(Format, Data)).

%% @spec diag(S) -> ok
%%       S = string()
%% @doc Print a debug/status message related to the test suite.
diag(S) -> msg("# " ++ S).

%% @spec diag(Format, Data) -> ok
%%      Format = atom() | string() | binary()
%%      Data = [term()]
%%      UnicodeList = [Unicode]
%%      Unicode = int()
%% @doc Print a debug/status message related to the test suite.
%% Function arguments are passed through io_lib:format/2.
diag(Format, Data) -> diag(io_lib:format(Format, Data)).

%% @spec expectation_mismatch_message(Got, Expected, Desc) -> ok
%%       Got = any()
%%       Expected = any()
%%       Desc = string()
%% @doc Print an expectation mismatch message in the test output.
expectation_mismatch_message(Got, Expected, Desc) ->
    msg("    ---"),
    msg("    description: ~p", [Desc]),
    msg("    found:       ~p", [Got]),
    msg("    wanted:      ~p", [Expected]),
    msg("    ..."),
    ok.

% @spec evaluate(Pass, Got, Expected, Desc) -> Result
%%       Pass = true | false
%%       Got = any()
%%       Expected = any()
%%       Desc = string()
%%       Result = true | false
%% @doc Evaluate a test statement, printing an expectation mismatch message
%%       if the test failed.
evaluate(Pass, Got, Expected, Desc) ->
    case mk_tap(Pass, Desc) of
        false ->
            expectation_mismatch_message(Got, Expected, Desc),
            false;
        true ->
            true
    end.

%% @spec ok(Expr, Desc) -> Result
%%       Expr = true | false
%%       Desc = string()
%%       Result = true | false
%% @doc Assert that a statement is true.
ok(Expr, Desc) -> evaluate(Expr == true, Expr, true, Desc).

%% @spec not_ok(Expr, Desc) -> Result
%%       Expr = true | false
%%       Desc = string()
%%       Result = true | false
%% @doc Assert that a statement is false.
not_ok(Expr, Desc) -> evaluate(Expr == false, Expr, false, Desc).

%% @spec is_ok(Expr, Desc) -> Result
%%       Expr = any()
%%       Desc = string()
%%       Result = true | false
%% @doc Assert that two values are the same.
is_ok(Expr, Desc) -> evaluate(Expr == ok, Expr, ok, Desc).

%% @spec is(Got, Expected, Desc) -> Result
%%       Got = any()
%%       Expected = any()
%%       Desc = string()
%%       Result = true | false
%% @doc Assert that two values are the same.
is(Got, Expected, Desc) -> evaluate(Got == Expected, Got, Expected, Desc).

%% @spec isnt(Got, Expected, Desc) -> Result
%%       Got = any()
%%       Expected = any()
%%       Desc = string()
%%       Result = true | false
%% @doc Assert that two values are not the same.
isnt(Got, Expected, Desc) -> evaluate(Got /= Expected, Got, Expected, Desc).

%% @spec is_greater(ValueA, ValueB, Desc) -> Result
%%       ValueA = number()
%%       ValueB = number()
%%       Desc = string()
%%       Result = true | false
%% @doc Assert that an integer is greater than another.
is_greater(ValueA, ValueB, Desc) when is_integer(ValueA), is_integer(ValueB) ->
    mk_tap(ValueA > ValueB, Desc).

%% @spec any(Got, Items, Desc) -> Result
%%       Got = any()
%%       Items = [any()]
%%       Desc = string()
%%       Result = true | false
%% @doc Assert that an item is in a list.
any(Got, Items, Desc) when is_function(Got) ->
    is(lists:any(Got, Items), true, Desc);
any(Got, Items, Desc) ->
    is(lists:member(Got, Items), true, Desc).

%% @spec none(Got, Items, Desc) -> Result
%%       Got = any()
%%       Items = [any()]
%%       Desc = string()
%%       Result = true | false
%% @doc Assert that an item is not in a list.
none(Got, Items, Desc) when is_function(Got) ->
    is(lists:any(Got, Items), false, Desc);
none(Got, Items, Desc) ->
    is(lists:member(Got, Items), false, Desc).

%% @spec fun_is(Fun, Expected, Desc) -> Result
%%       Fun = function()
%%       Expected = any()
%%       Desc = string()
%%       Result = true | false
%% @doc Use an anonymous function to assert a pattern match.
fun_is(Fun, Expected, Desc) when is_function(Fun) ->
    is(Fun(Expected), true, Desc).

%% @spec expect_fun(ExpectFun, Got, Desc) -> Result
%%       ExpectFun = function()
%%       Got = any()
%%       Desc = string()
%%       Result = true | false
%% @doc Use an anonymous function to assert a pattern match, using actual
%%       value as the argument to the function.
expect_fun(ExpectFun, Got, Desc) ->
    evaluate(ExpectFun(Got), Got, ExpectFun, Desc).

%% @spec expect_fun(ExpectFun, Got, Desc, ExpectStr) -> Result
%%       ExpectFun = function()
%%       Got = any()
%%       Desc = string()
%%       ExpectStr = string()
%%       Result = true | false
%% @doc Use an anonymous function to assert a pattern match, using actual
%%       value as the argument to the function.
expect_fun(ExpectFun, Got, Desc, ExpectStr) ->
    evaluate(ExpectFun(Got), Got, ExpectStr, Desc).

%% @equiv skip(TestFun, "")
skip(TestFun) when is_function(TestFun) ->
    skip(TestFun, "").

%% @spec skip(TestFun, Reason) -> ok
%%       TestFun = function()
%%       Reason = string()
%% @doc Skip a test.
skip(TestFun, Reason) when is_function(TestFun), is_list(Reason) ->
    begin_skip(Reason),
    catch TestFun(),
    end_skip(),
    ok.

%% @spec skip(Q, TestFun, Reason) -> ok
%%       Q = true | false | function()
%%       TestFun = function()
%%       Reason = string()
%% @doc Skips a test conditionally. The first argument to this function can
%% either be the 'true' or 'false' atoms or a function that returns 'true' or
%% 'false'.
skip(QFun, TestFun, Reason) when is_function(QFun), is_function(TestFun), is_list(Reason) ->
    case QFun() of
        true -> begin_skip(Reason), TestFun(), end_skip();
        _ -> TestFun()
    end,
    ok;

skip(Q, TestFun, Reason) when is_function(TestFun), is_list(Reason), Q == true ->
    begin_skip(Reason),
    TestFun(),
    end_skip(),
    ok;

skip(_, TestFun, Reason) when is_function(TestFun), is_list(Reason) ->
    TestFun(),
    ok.

%% @private
begin_skip(Reason) ->
    etap_server ! {self(), begin_skip, Reason}.

%% @private
end_skip() ->
    etap_server ! {self(), end_skip}.

%% @spec contains_ok(string(), string(), string()) -> true | false
%% @doc Assert that a string is contained in another string.
contains_ok(Source, String, Desc) ->
    etap:isnt(
        string:str(Source, String),
        0,
        Desc
    ).

%% @spec is_before(string(), string(), string(), string()) -> true | false
%% @doc Assert that a string comes before another string within a larger body.
is_before(Source, StringA, StringB, Desc) ->
    etap:is_greater(
        string:str(Source, StringB),
        string:str(Source, StringA),
        Desc
    ).

%% @doc Assert that a given variable is a pid.
is_pid(Pid, Desc) when is_pid(Pid) -> etap:ok(true, Desc);
is_pid(_, Desc) -> etap:ok(false, Desc).

%% @doc Assert that a given process/pid is alive.
is_alive(Pid, Desc) ->
    etap:ok(erlang:is_process_alive(Pid), Desc).

%% @doc Assert that the current function of a pid is a given {M, F, A} tuple.
is_mfa(Pid, MFA, Desc) ->
    etap:is({current_function, MFA}, erlang:process_info(Pid, current_function), Desc).

%% @spec loaded_ok(atom(), string()) -> true | false
%% @doc Assert that a module has been loaded successfully.
loaded_ok(M, Desc) when is_atom(M) ->
    etap:fun_is(fun({module, _}) -> true; (_) -> false end, code:load_file(M), Desc).

%% @spec can_ok(atom(), atom()) -> true | false
%% @doc Assert that a module exports a given function.
can_ok(M, F) when is_atom(M), is_atom(F) ->
    Matches = [X || {X, _} <- M:module_info(exports), X == F],
    etap:ok(Matches > 0, lists:concat([M, " can ", F])).

%% @spec can_ok(atom(), atom(), integer()) -> true | false
%% @doc Assert that a module exports a given function with a given arity.
can_ok(M, F, A) when is_atom(M); is_atom(F), is_number(A) ->
    Matches = [X || X <- M:module_info(exports), X == {F, A}],
    etap:ok(Matches > 0, lists:concat([M, " can ", F, "/", A])).

%% @spec has_attrib(M, A) -> true | false
%%       M = atom()
%%       A = atom()
%% @doc Asserts that a module has a given attribute.
has_attrib(M, A) when is_atom(M), is_atom(A) ->
    etap:isnt(
        proplists:get_value(A, M:module_info(attributes), 'asdlkjasdlkads'),
        'asdlkjasdlkads',
        lists:concat([M, " has attribute ", A])
    ).

%% @spec has_attrib(M, A. V) -> true | false
%%       M = atom()
%%       A = atom()
%%       V = any()
%% @doc Asserts that a module has a given attribute with a given value.
is_attrib(M, A, V) when is_atom(M) andalso is_atom(A) ->
    etap:is(
        proplists:get_value(A, M:module_info(attributes)),
        [V],
        lists:concat([M, "'s ", A, " is ", V])
    ).

%% @spec is_behavior(M, B) -> true | false
%%       M = atom()
%%       B = atom()
%% @doc Asserts that a given module has a specific behavior.
is_behaviour(M, B) when is_atom(M) andalso is_atom(B) ->
    is_attrib(M, behaviour, B).

%% @doc Assert that an exception is raised when running a given function.
dies_ok(F, Desc) ->
    case (catch F()) of
        {'EXIT', _} -> etap:ok(true, Desc);
        _ -> etap:ok(false, Desc)
    end.

%% @doc Assert that an exception is not raised when running a given function.
lives_ok(F, Desc) ->
    etap:is(try_this(F), success, Desc).

%% @doc Assert that the exception thrown by a function matches the given exception.
throws_ok(F, Exception, Desc) ->
    try F() of
        _ -> etap:ok(nok, Desc)
    catch
        _:E ->
            etap:is(E, Exception, Desc)
    end.

%% @private
%% @doc Run a function and catch any exceptions.
try_this(F) when is_function(F, 0) ->
    try F() of
        _ -> success
    catch
        throw:E -> {throw, E};
        error:E -> {error, E};
        exit:E -> {exit, E}
    end.

%% @private
%% @doc Start the etap_server process if it is not running already.
ensure_test_server() ->
    case whereis(etap_server) of
        undefined ->
            proc_lib:start(?MODULE, start_etap_server,[]);
        _ ->
            diag("The test server is already running.")
    end.

%% @private
%% @doc Start the etap_server loop and register itself as the etap_server
%% process.
start_etap_server() ->
    catch register(etap_server, self()),
    proc_lib:init_ack(ok),
    etap:test_server(#test_state{
        planned = 0,
        count = 0,
        pass = 0,
        fail = 0,
        skip = 0,
        skip_reason = ""
    }).


%% @private
%% @doc The main etap_server receive/run loop. The etap_server receive loop
%% responds to seven messages apperatining to failure or passing of tests.
%% It is also used to initiate the testing process with the {_, plan, _}
%% message that clears the current test state.
test_server(State) ->
    NewState = receive
        {_From, plan, unknown} ->
            io:format("# Current time local ~s~n", [datetime(erlang:localtime())]),
            io:format("# Using etap version ~p~n", [ proplists:get_value(vsn, proplists:get_value(attributes, etap:module_info())) ]),
            State#test_state{
                planned = -1,
                count = 0,
                pass = 0,
                fail = 0,
                skip = 0,
                skip_reason = ""
            };
        {_From, plan, N} ->
            io:format("# Current time local ~s~n", [datetime(erlang:localtime())]),
            io:format("# Using etap version ~p~n", [ proplists:get_value(vsn, proplists:get_value(attributes, etap:module_info())) ]),
            io:format("1..~p~n", [N]),
            State#test_state{
                planned = N,
                count = 0,
                pass = 0,
                fail = 0,
                skip = 0,
                skip_reason = ""
            };
        {_From, begin_skip, Reason} ->
            State#test_state{
                skip = 1,
                skip_reason = Reason
            };
        {_From, end_skip} ->
            State#test_state{
                skip = 0,
                skip_reason = ""
            };
        {_From, pass, Desc} ->
            FullMessage = skip_diag(
                " - " ++ Desc,
                State#test_state.skip,
                State#test_state.skip_reason
            ),
            io:format("ok ~p ~s~n", [State#test_state.count + 1, FullMessage]),
            State#test_state{
                count = State#test_state.count + 1,
                pass = State#test_state.pass + 1
            };

        {_From, fail, Desc} ->
            FullMessage = skip_diag(
                " - " ++ Desc,
                State#test_state.skip,
                State#test_state.skip_reason
            ),
            io:format("not ok ~p ~s~n", [State#test_state.count + 1, FullMessage]),
            State#test_state{
                count = State#test_state.count + 1,
                fail = State#test_state.fail + 1
            };
        {From, state} ->
            From ! State,
            State;
        {_From, diag, Message} ->
            io:format("~s~n", [Message]),
            State;
        {From, count} ->
            From ! State#test_state.count,
            State;
        {From, is_skip} ->
            From ! State#test_state.skip,
            State;
        done ->
            exit(normal)
    end,
    test_server(NewState).

%% @private
%% @doc Process the result of a test and send it to the etap_server process.
mk_tap(Result, Desc) ->
    IsSkip = lib:sendw(etap_server, is_skip),
    case [IsSkip, Result] of
        [_, true] ->
            etap_server ! {self(), pass, Desc},
            true;
        [1, _] ->
            etap_server ! {self(), pass, Desc},
            true;
        _ ->
            etap_server ! {self(), fail, Desc},
            false
    end.

%% @private
%% @doc Format a date/time string.
datetime(DateTime) ->
    {{Year, Month, Day}, {Hour, Min, Sec}} = DateTime,
    io_lib:format("~4.10.0B-~2.10.0B-~2.10.0B ~2.10.0B:~2.10.0B:~2.10.0B", [Year, Month, Day, Hour, Min, Sec]).

%% @private
%% @doc Craft an output message taking skip/todo into consideration.
skip_diag(Message, 0, _) ->
    Message;
skip_diag(_Message, 1, "") ->
    " # SKIP";
skip_diag(_Message, 1, Reason) ->
    " # SKIP : " ++ Reason.