diff --git a/src/lager_handler_watcher.erl b/src/lager_handler_watcher.erl index 4e595e4..1f4cc06 100644 --- a/src/lager_handler_watcher.erl +++ b/src/lager_handler_watcher.erl @@ -48,6 +48,7 @@ start(Sink, Module, Config) -> gen_server:start(?MODULE, [Sink, Module, Config], []). init([Sink, Module, Config]) -> + process_flag(trap_exit, true), install_handler(Sink, Module, Config), {ok, #state{sink=Sink, module=Module, config=Config}}. @@ -62,7 +63,7 @@ handle_info({gen_event_EXIT, Module, normal}, #state{module=Module} = State) -> handle_info({gen_event_EXIT, Module, shutdown}, #state{module=Module} = State) -> {stop, normal, State}; handle_info({gen_event_EXIT, Module, {'EXIT', {kill_me, [_KillerHWM, KillerReinstallAfter]}}}, - #state{module=Module, sink=Sink} = State) -> + #state{module=Module, sink=Sink, config = Config} = State) -> %% Brutally kill the manager but stay alive to restore settings. %% %% SinkPid here means the gen_event process. Handlers *all* live inside the @@ -70,9 +71,11 @@ handle_info({gen_event_EXIT, Module, {'EXIT', {kill_me, [_KillerHWM, KillerReins %% pending log messages in its mailbox will die too. SinkPid = whereis(Sink), unlink(SinkPid), + {message_queue_len, Len} = process_info(SinkPid, message_queue_len), + error_logger:error_msg("Killing sink ~p, current message_queue_len:~p~n", [Sink, Len]), exit(SinkPid, kill), - erlang:send_after(KillerReinstallAfter, self(), {reboot, Sink}), - {noreply, State}; + timer:apply_after(KillerReinstallAfter, lager_app, start_handler, [Sink, Module, Config]), + {stop, normal, State}; handle_info({gen_event_EXIT, Module, Reason}, #state{module=Module, config=Config, sink=Sink} = State) -> case lager:log(error, self(), "Lager event handler ~p exited with reason ~s", @@ -92,6 +95,10 @@ handle_info({reboot, Sink}, State) -> {noreply, State}; handle_info(stop, State) -> {stop, normal, State}; +handle_info({'EXIT', _Pid, killed}, #state{module=Module, config=Config, sink=Sink} = State) -> + Tmr = application:get_env(lager, killer_reinstall_after, 5000), + timer:apply_after(Tmr, lager_app, start_handler, [Sink, Module, Config]), + {stop, normal, State}; handle_info(_Info, State) -> {noreply, State}. @@ -192,6 +199,33 @@ reinstall_on_runtime_failure_test_() -> ] }. +reinstall_handlers_after_killer_hwm_test_() -> + {timeout, 60000, + [ + fun() -> + error_logger:tty(false), + application:load(lager), + application:set_env(lager, handlers, [{lager_manager_killer, [1000, 5000]}]), + application:set_env(lager, error_logger_redirect, false), + application:set_env(lager, killer_reinstall_after, 5000), + application:unset_env(lager, crash_log), + lager:start(), + lager:trace_file("foo", [{foo, "bar"}], error), + L = length(gen_event:which_handlers(lager_event)), + try + lager_manager_killer:kill_me(), + timer:sleep(6000), + ?assertEqual(L, length(gen_event:which_handlers(lager_event))), + file:delete("foo") + after + application:stop(lager), + application:stop(goldrush), + error_logger:tty(true) + end + end + ] + }. + pop_until(String, Fun) -> try_backend_pop(lager_test_backend:pop(), String, Fun).