From 49311b27e1309136dbd58db3760f7a207e354be2 Mon Sep 17 00:00:00 2001 From: "Brian L. Troutwine" Date: Thu, 10 Mar 2016 16:03:39 -0800 Subject: [PATCH] Attempt to redress test failures. At this point in the work, the killer will correctly stop the DEFAULT_SINK in the event of overload but our friend never comes back up and lager stops working past this point. Oops! Error message: =SUPERVISOR REPORT==== 10-Mar-2016::16:18:11 === Supervisor: {local,lager_handler_watcher_sup} Context: child_terminated Reason: killed Offender: [{pid,<0.63.0>}, {id,lager_handler_watcher}, {mfargs,{lager_handler_watcher,start_link,undefined}}, {restart_type,temporary}, {shutdown,5000}, {child_type,worker}] =SUPERVISOR REPORT==== 10-Mar-2016::16:18:11 === Supervisor: {local,lager_handler_watcher_sup} Context: child_terminated Reason: killed Offender: [{pid,<0.59.0>}, {id,lager_handler_watcher}, {mfargs,{lager_handler_watcher,start_link,undefined}}, {restart_type,temporary}, {shutdown,5000}, {child_type,worker}] =PROGRESS REPORT==== 10-Mar-2016::16:18:11 === supervisor: {local,lager_sup} started: [{pid,<0.113.0>}, {id,lager}, {mfargs,{gen_event,start_link,[{local,lager_event}]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] Signed-off-by: Brian L. Troutwine --- src/lager_app.erl | 45 +++++++++++++----------------- src/lager_handler_watcher.erl | 8 +++--- src/lager_manager_killer.erl | 38 ++++++++++++------------- test/lager_manager_killer_test.erl | 4 +-- test/lager_slow_backend.erl | 2 +- 5 files changed, 45 insertions(+), 52 deletions(-) diff --git a/src/lager_app.erl b/src/lager_app.erl index d40f85b..2661eac 100644 --- a/src/lager_app.erl +++ b/src/lager_app.erl @@ -151,6 +151,25 @@ start_error_logger_handler(_, HWM, {ok, WhiteList}) -> throw({error, bad_config}) end, + %% Conditionally start the high watermark killer. + case application:get_env(lager, killer_hwm) of + undefined -> ok; + {ok, undefined} -> ok; + {ok, KillerHWM} when is_integer(KillerHWM), KillerHWM >= 0 -> + KillerReinstallAfter = case application:get_env(lager, killer_reinstall_after) of + undefined -> 5000; + {ok, undefined} -> 5000; + {ok, V} when is_integer(V), V >= 0 -> V; + {ok, BadKillerReinstallAfter} -> + error_logger:error_msg("Invalid value for 'cooldown': ~p~n", + [BadKillerReinstallAfter]), + throw({error, bad_config}) + end, + _ = supervisor:start_child(lager_handler_watcher_sup, + [?DEFAULT_SINK, lager_manager_killer, + [KillerHWM, KillerReinstallAfter]]) + end, + case supervisor:start_child(lager_handler_watcher_sup, [error_logger, error_logger_lager_h, [HWM, GlStrategy]]) of {ok, _} -> [begin error_logger:delete_report_handler(X), X end || @@ -159,32 +178,6 @@ start_error_logger_handler(_, HWM, {ok, WhiteList}) -> [] end, - case application:get_env(lager, killer_hwm) of - undefined -> - ok; - {ok, undefined} -> - undefined; - {ok, KillerHWM} when is_integer(KillerHWM), KillerHWM >= 0 -> - KillerReinstallAfter = - case application:get_env(lager, killer_reinstall_after) of - undefined -> - 5000; - {ok, undefined} -> - 5000; - {ok, V} when is_integer(V), V >= 0 -> - V; - {ok, BadKillerReinstallAfter} -> - error_logger:error_msg("Invalid value for 'cooldown': ~p~n", [BadKillerReinstallAfter]), - throw({error, bad_config}) - end, - _ = supervisor:start_child(lager_handler_watcher_sup, - [lager_event, lager_manager_killer, [KillerHWM, KillerReinstallAfter]]), - ok; - {ok, BadKillerHWM} -> - error_logger:error_msg("Invalid value for 'floodline': ~p~n", [BadKillerHWM]), - throw({error, bad_config}) - end, - Handlers = case application:get_env(lager, handlers) of undefined -> [{lager_console_backend, info}, diff --git a/src/lager_handler_watcher.erl b/src/lager_handler_watcher.erl index c051101..da67d8b 100644 --- a/src/lager_handler_watcher.erl +++ b/src/lager_handler_watcher.erl @@ -61,14 +61,14 @@ handle_info({gen_event_EXIT, Module, normal}, #state{module=Module} = State) -> {stop, normal, State}; handle_info({gen_event_EXIT, Module, shutdown}, #state{module=Module} = State) -> {stop, normal, State}; -handle_info({gen_event_EXIT, Module, {'EXIT', {kill_me, [KillerHWM, KillerReinstallAfter]}}}, - #state{module=Module, event=Event} = State) -> +handle_info({gen_event_EXIT, Module, {'EXIT', {kill_me, [_KillerHWM, KillerReinstallAfter]}}}, + #state{module=Module, sink=Sink} = State) -> % Brutally kill the manager but stay alive to restore settings. - Manager = whereis(Event), + Manager = whereis(Sink), unlink(Manager), exit(Manager, kill), erlang:send_after(KillerReinstallAfter, self(), reinstall_handler), - {noreply, State#state{config=[KillerHWM, KillerReinstallAfter]}}; + {noreply, State}; handle_info({gen_event_EXIT, Module, Reason}, #state{module=Module, config=Config, sink=Sink} = State) -> case lager:log(error, self(), "Lager event handler ~p exited with reason ~s", diff --git a/src/lager_manager_killer.erl b/src/lager_manager_killer.erl index bb959f9..158b471 100644 --- a/src/lager_manager_killer.erl +++ b/src/lager_manager_killer.erl @@ -7,38 +7,38 @@ -include("lager.hrl"). -record(state, { - killer_hwm :: non_neg_integer(), - killer_reinstall_after :: non_neg_integer() -}). + killer_hwm :: non_neg_integer(), + killer_reinstall_after :: non_neg_integer() + }). init([KillerHWM, KillerReinstallAfter]) -> - {ok, #state{killer_hwm=KillerHWM, killer_reinstall_after=KillerReinstallAfter}}. + {ok, #state{killer_hwm=KillerHWM, killer_reinstall_after=KillerReinstallAfter}}. handle_call(get_loglevel, State) -> - {ok, {mask, ?LOG_NONE}, State}; + {ok, {mask, ?LOG_NONE}, State}; handle_call({set_loglevel, _Level}, State) -> - {ok, ok, State}; + {ok, ok, State}; handle_call(get_settings, State = #state{killer_hwm=KillerHWM, killer_reinstall_after=KillerReinstallAfter}) -> - {ok, [KillerHWM, KillerReinstallAfter], State}; + {ok, [KillerHWM, KillerReinstallAfter], State}; handle_call(_Request, State) -> - {ok, ok, State}. + {ok, ok, State}. handle_event({log, _Message}, State = #state{killer_hwm=KillerHWM, killer_reinstall_after=KillerReinstallAfter}) -> - {message_queue_len, Len} = process_info(self(), message_queue_len), - case Len > KillerHWM of - true -> - exit({kill_me, [KillerHWM, KillerReinstallAfter]}); - _ -> - {ok, State} - end; + {message_queue_len, Len} = process_info(self(), message_queue_len), + case Len > KillerHWM of + true -> + exit({kill_me, [KillerHWM, KillerReinstallAfter]}); + _ -> + {ok, State} + end; handle_event(_Event, State) -> - {ok, State}. + {ok, State}. handle_info(_Info, State) -> - {ok, State}. + {ok, State}. terminate(_Reason, _State) -> - ok. + ok. code_change(_OldVsn, State, _Extra) -> - {ok, State}. + {ok, State}. diff --git a/test/lager_manager_killer_test.erl b/test/lager_manager_killer_test.erl index 1fcb59f..6e63079 100644 --- a/test/lager_manager_killer_test.erl +++ b/test/lager_manager_killer_test.erl @@ -12,7 +12,7 @@ overload_test() -> Delay = 1000, % sleep 1 sec on every log KillerHWM = 10, % kill the manager if there are more than 10 pending logs KillerReinstallAfter = 1000, % reinstall killer after 1 sec - application:set_env(lager, handlers, [{lager_slow_backend, Delay}]), + application:set_env(lager, handlers, [{lager_slow_backend, [{delay, Delay}]}]), application:set_env(lager, async_threshold, undefined), application:set_env(lager, killer_hwm, KillerHWM), application:set_env(lager, killer_reinstall_after, KillerReinstallAfter), @@ -52,4 +52,4 @@ ensure_started(App) -> ensure_started(App) end. --endif. \ No newline at end of file +-endif. diff --git a/test/lager_slow_backend.erl b/test/lager_slow_backend.erl index 9771083..e467c23 100644 --- a/test/lager_slow_backend.erl +++ b/test/lager_slow_backend.erl @@ -10,7 +10,7 @@ delay :: non_neg_integer() }). -init(Delay) -> +init([{delay, Delay}]) -> {ok, #state{delay=Delay}}. handle_call(get_loglevel, State) ->