Вы не можете выбрать более 25 тем
Темы должны начинаться с буквы или цифры, могут содержать дефисы(-) и должны содержать не более 35 символов.
242 строки
9.2 KiB
242 строки
9.2 KiB
%% Copyright (c) 2011-2012 Basho Technologies, Inc. All Rights Reserved.
|
|
%%
|
|
%% This file is provided to you under the Apache License,
|
|
%% Version 2.0 (the "License"); you may not use this file
|
|
%% except in compliance with the License. You may obtain
|
|
%% a copy of the License at
|
|
%%
|
|
%% http://www.apache.org/licenses/LICENSE-2.0
|
|
%%
|
|
%% Unless required by applicable law or agreed to in writing,
|
|
%% software distributed under the License is distributed on an
|
|
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
%% KIND, either express or implied. See the License for the
|
|
%% specific language governing permissions and limitations
|
|
%% under the License.
|
|
|
|
%% @doc A process that does a gen_event:add_sup_handler and attempts to re-add
|
|
%% event handlers when they exit.
|
|
|
|
%% @private
|
|
|
|
-module(lager_handler_watcher).
|
|
|
|
-behaviour(gen_server).
|
|
|
|
-include("lager.hrl").
|
|
|
|
-ifdef(TEST).
|
|
-include_lib("eunit/include/eunit.hrl").
|
|
-endif.
|
|
|
|
%% callbacks
|
|
-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2,
|
|
code_change/3]).
|
|
|
|
-export([start_link/3, start/3]).
|
|
|
|
-record(state, {
|
|
module :: atom(),
|
|
config :: any(),
|
|
sink :: pid() | atom()
|
|
}).
|
|
|
|
start_link(Sink, Module, Config) ->
|
|
gen_server:start_link(?MODULE, [Sink, Module, Config], []).
|
|
|
|
start(Sink, Module, Config) ->
|
|
gen_server:start(?MODULE, [Sink, Module, Config], []).
|
|
|
|
init([Sink, Module, Config]) ->
|
|
process_flag(trap_exit, true),
|
|
install_handler(Sink, Module, Config),
|
|
{ok, #state{sink=Sink, module=Module, config=Config}}.
|
|
|
|
handle_call(_Call, _From, State) ->
|
|
{reply, ok, State}.
|
|
|
|
handle_cast(_Request, State) ->
|
|
{noreply, State}.
|
|
|
|
handle_info({gen_event_EXIT, Module, normal}, #state{module=Module} = State) ->
|
|
{stop, normal, State};
|
|
handle_info({gen_event_EXIT, Module, shutdown}, #state{module=Module} = State) ->
|
|
{stop, normal, State};
|
|
handle_info({gen_event_EXIT, Module, {'EXIT', {kill_me, [_KillerHWM, KillerReinstallAfter]}}},
|
|
#state{module=Module, sink=Sink, config = Config} = State) ->
|
|
%% Brutally kill the manager but stay alive to restore settings.
|
|
%%
|
|
%% SinkPid here means the gen_event process. Handlers *all* live inside the
|
|
%% same gen_event process space, so when the Pid is killed, *all* of the
|
|
%% pending log messages in its mailbox will die too.
|
|
SinkPid = whereis(Sink),
|
|
unlink(SinkPid),
|
|
{message_queue_len, Len} = process_info(SinkPid, message_queue_len),
|
|
error_logger:error_msg("Killing sink ~p, current message_queue_len:~p~n", [Sink, Len]),
|
|
exit(SinkPid, kill),
|
|
timer:apply_after(KillerReinstallAfter, lager_app, start_handler, [Sink, Module, Config]),
|
|
{stop, normal, State};
|
|
handle_info({gen_event_EXIT, Module, Reason}, #state{module=Module,
|
|
config=Config, sink=Sink} = State) ->
|
|
case lager:log(error, self(), "Lager event handler ~p exited with reason ~s",
|
|
[Module, error_logger_lager_h:format_reason(Reason)]) of
|
|
ok ->
|
|
install_handler(Sink, Module, Config);
|
|
{error, _} ->
|
|
%% lager is not working, so installing a handler won't work
|
|
ok
|
|
end,
|
|
{noreply, State};
|
|
handle_info(reinstall_handler, #state{module=Module, config=Config, sink=Sink} = State) ->
|
|
install_handler(Sink, Module, Config),
|
|
{noreply, State};
|
|
handle_info({reboot, Sink}, State) ->
|
|
_ = lager_app:boot(Sink),
|
|
{noreply, State};
|
|
handle_info(stop, State) ->
|
|
{stop, normal, State};
|
|
handle_info({'EXIT', _Pid, killed}, #state{module=Module, config=Config, sink=Sink} = State) ->
|
|
Tmr = application:get_env(lager, killer_reinstall_after, 5000),
|
|
timer:apply_after(Tmr, lager_app, start_handler, [Sink, Module, Config]),
|
|
{stop, normal, State};
|
|
handle_info(_Info, State) ->
|
|
{noreply, State}.
|
|
|
|
terminate(_Reason, _State) ->
|
|
ok.
|
|
|
|
code_change(_OldVsn, State, _Extra) ->
|
|
{ok, State}.
|
|
|
|
%% internal
|
|
install_handler(Sink, lager_backend_throttle, Config) ->
|
|
%% The lager_backend_throttle needs to know to which sink it is
|
|
%% attached, hence this admittedly ugly workaround. Handlers are
|
|
%% sensitive to the structure of the configuration sent to `init',
|
|
%% sadly, so it's not trivial to add a configuration item to be
|
|
%% ignored to backends without breaking 3rd party handlers.
|
|
install_handler2(Sink, lager_backend_throttle, [{sink, Sink}|Config]);
|
|
install_handler(Sink, Module, Config) ->
|
|
install_handler2(Sink, Module, Config).
|
|
|
|
%% private
|
|
install_handler2(Sink, Module, Config) ->
|
|
case gen_event:add_sup_handler(Sink, Module, Config) of
|
|
ok ->
|
|
?INT_LOG(debug, "Lager installed handler ~p into ~p", [Module, Sink]),
|
|
lager:update_loglevel_config(Sink),
|
|
ok;
|
|
{error, {fatal, Reason}} ->
|
|
?INT_LOG(error, "Lager fatally failed to install handler ~p into"
|
|
" ~p, NOT retrying: ~p", [Module, Sink, Reason]),
|
|
%% tell ourselves to stop
|
|
self() ! stop,
|
|
ok;
|
|
Error ->
|
|
%% try to reinstall it later
|
|
?INT_LOG(error, "Lager failed to install handler ~p into"
|
|
" ~p, retrying later : ~p", [Module, Sink, Error]),
|
|
erlang:send_after(5000, self(), reinstall_handler),
|
|
ok
|
|
end.
|
|
|
|
-ifdef(TEST).
|
|
|
|
from_now(Seconds) ->
|
|
{Mega, Secs, Micro} = os:timestamp(),
|
|
{Mega, Secs + Seconds, Micro}.
|
|
|
|
reinstall_on_initial_failure_test_() ->
|
|
{timeout, 60000,
|
|
[
|
|
fun() ->
|
|
error_logger:tty(false),
|
|
application:load(lager),
|
|
application:set_env(lager, handlers, [{lager_test_backend, info}, {lager_crash_backend, [from_now(2), undefined]}]),
|
|
application:set_env(lager, error_logger_redirect, false),
|
|
application:unset_env(lager, crash_log),
|
|
lager:start(),
|
|
try
|
|
{_Level, _Time, Message, _Metadata} = lager_test_backend:pop(),
|
|
?assertMatch("Lager failed to install handler lager_crash_backend into lager_event, retrying later :"++_, lists:flatten(Message)),
|
|
timer:sleep(6000),
|
|
lager_test_backend:flush(),
|
|
?assertEqual(0, lager_test_backend:count()),
|
|
?assert(lists:member(lager_crash_backend, gen_event:which_handlers(lager_event)))
|
|
after
|
|
application:stop(lager),
|
|
application:stop(goldrush),
|
|
error_logger:tty(true)
|
|
end
|
|
end
|
|
]
|
|
}.
|
|
|
|
reinstall_on_runtime_failure_test_() ->
|
|
{timeout, 60000,
|
|
[
|
|
fun() ->
|
|
error_logger:tty(false),
|
|
application:load(lager),
|
|
application:set_env(lager, handlers, [{lager_test_backend, info}, {lager_crash_backend, [undefined, from_now(5)]}]),
|
|
application:set_env(lager, error_logger_redirect, false),
|
|
application:unset_env(lager, crash_log),
|
|
lager:start(),
|
|
try
|
|
?assert(lists:member(lager_crash_backend, gen_event:which_handlers(lager_event))),
|
|
timer:sleep(6000),
|
|
|
|
pop_until("Lager event handler lager_crash_backend exited with reason crash", fun lists:flatten/1),
|
|
pop_until("Lager failed to install handler lager_crash_backend into lager_event, retrying later",
|
|
fun(Msg) -> string:substr(lists:flatten(Msg), 1, 84) end),
|
|
?assertEqual(false, lists:member(lager_crash_backend, gen_event:which_handlers(lager_event)))
|
|
after
|
|
application:stop(lager),
|
|
application:stop(goldrush),
|
|
error_logger:tty(true)
|
|
end
|
|
end
|
|
]
|
|
}.
|
|
|
|
reinstall_handlers_after_killer_hwm_test_() ->
|
|
{timeout, 60000,
|
|
[
|
|
fun() ->
|
|
error_logger:tty(false),
|
|
application:load(lager),
|
|
application:set_env(lager, handlers, [{lager_manager_killer, [1000, 5000]}]),
|
|
application:set_env(lager, error_logger_redirect, false),
|
|
application:set_env(lager, killer_reinstall_after, 5000),
|
|
application:unset_env(lager, crash_log),
|
|
lager:start(),
|
|
lager:trace_file("foo", [{foo, "bar"}], error),
|
|
L = length(gen_event:which_handlers(lager_event)),
|
|
try
|
|
lager_manager_killer:kill_me(),
|
|
timer:sleep(6000),
|
|
?assertEqual(L, length(gen_event:which_handlers(lager_event))),
|
|
file:delete("foo")
|
|
after
|
|
application:stop(lager),
|
|
application:stop(goldrush),
|
|
error_logger:tty(true)
|
|
end
|
|
end
|
|
]
|
|
}.
|
|
|
|
pop_until(String, Fun) ->
|
|
try_backend_pop(lager_test_backend:pop(), String, Fun).
|
|
|
|
try_backend_pop(undefined, String, _Fun) ->
|
|
throw("Not found: " ++ String);
|
|
try_backend_pop({_Severity, _Date, Msg, _Metadata}, String, Fun) ->
|
|
case Fun(Msg) of
|
|
String ->
|
|
ok;
|
|
_ ->
|
|
try_backend_pop(lager_test_backend:pop(), String, Fun)
|
|
end.
|
|
|
|
-endif.
|