Attempt to redress test failures.

At this point in the work, the killer will correctly stop the
DEFAULT_SINK in the event of overload but our friend never comes back
up and lager stops working past this point.

Oops!

Error message:

=SUPERVISOR REPORT==== 10-Mar-2016::16:18:11 ===
     Supervisor: {local,lager_handler_watcher_sup}
     Context:    child_terminated
     Reason:     killed
     Offender:   [{pid,<0.63.0>},
                  {id,lager_handler_watcher},
                  {mfargs,{lager_handler_watcher,start_link,undefined}},
                  {restart_type,temporary},
                  {shutdown,5000},
                  {child_type,worker}]

=SUPERVISOR REPORT==== 10-Mar-2016::16:18:11 ===
     Supervisor: {local,lager_handler_watcher_sup}
     Context:    child_terminated
     Reason:     killed
     Offender:   [{pid,<0.59.0>},
                  {id,lager_handler_watcher},
                  {mfargs,{lager_handler_watcher,start_link,undefined}},
                  {restart_type,temporary},
                  {shutdown,5000},
                  {child_type,worker}]

=PROGRESS REPORT==== 10-Mar-2016::16:18:11 ===
          supervisor: {local,lager_sup}
             started: [{pid,<0.113.0>},
                       {id,lager},
                       {mfargs,{gen_event,start_link,[{local,lager_event}]}},
                       {restart_type,permanent},
                       {shutdown,5000},
                       {child_type,worker}]

Signed-off-by: Brian L. Troutwine <brian@troutwine.us>

src/lager_app.erl

@@ -151,6 +151,25 @@ start_error_logger_handler(_, HWM, {ok, WhiteList}) ->
                         throw({error, bad_config})
                 end,
 
+    %% Conditionally start the high watermark killer.
+    case application:get_env(lager, killer_hwm) of
+        undefined -> ok;
+        {ok, undefined} -> ok;
+        {ok, KillerHWM} when is_integer(KillerHWM), KillerHWM >= 0 ->
+            KillerReinstallAfter = case application:get_env(lager, killer_reinstall_after) of
+                                       undefined -> 5000;
+                                       {ok, undefined} -> 5000;
+                                       {ok, V} when is_integer(V), V >= 0 -> V;
+                                       {ok, BadKillerReinstallAfter} ->
+                                           error_logger:error_msg("Invalid value for 'cooldown': ~p~n",
+                                                                  [BadKillerReinstallAfter]),
+                                           throw({error, bad_config})
+                                   end,
+            _ = supervisor:start_child(lager_handler_watcher_sup,
+                                       [?DEFAULT_SINK, lager_manager_killer,
+                                        [KillerHWM, KillerReinstallAfter]])
+    end,
+
     case supervisor:start_child(lager_handler_watcher_sup, [error_logger, error_logger_lager_h, [HWM, GlStrategy]]) of
         {ok, _} ->
             [begin error_logger:delete_report_handler(X), X end ||
@@ -159,32 +178,6 @@ start_error_logger_handler(_, HWM, {ok, WhiteList}) ->
             []
     end,
 
-    case application:get_env(lager, killer_hwm) of
-        undefined ->
-            ok;
-        {ok, undefined} ->
-            undefined;
-        {ok, KillerHWM} when is_integer(KillerHWM), KillerHWM >= 0 ->
-            KillerReinstallAfter =
-            case application:get_env(lager, killer_reinstall_after) of
-                undefined ->
-                    5000;
-                {ok, undefined} ->
-                    5000;
-                {ok, V} when is_integer(V), V >= 0 ->
-                    V;
-                {ok, BadKillerReinstallAfter} ->
-                    error_logger:error_msg("Invalid value for 'cooldown': ~p~n", [BadKillerReinstallAfter]),
-                    throw({error, bad_config})
-            end,
-            _ = supervisor:start_child(lager_handler_watcher_sup,
-                    [lager_event, lager_manager_killer, [KillerHWM, KillerReinstallAfter]]),
-            ok;
-        {ok, BadKillerHWM} ->
-            error_logger:error_msg("Invalid value for 'floodline': ~p~n", [BadKillerHWM]),
-            throw({error, bad_config})
-    end,
-
     Handlers = case application:get_env(lager, handlers) of
         undefined ->
             [{lager_console_backend, info},

src/lager_handler_watcher.erl

@@ -61,14 +61,14 @@ handle_info({gen_event_EXIT, Module, normal}, #state{module=Module} = State) ->
     {stop, normal, State};
 handle_info({gen_event_EXIT, Module, shutdown}, #state{module=Module} = State) ->
     {stop, normal, State};
-handle_info({gen_event_EXIT, Module, {'EXIT', {kill_me, [KillerHWM, KillerReinstallAfter]}}},
-        #state{module=Module, event=Event} = State) ->
+handle_info({gen_event_EXIT, Module, {'EXIT', {kill_me, [_KillerHWM, KillerReinstallAfter]}}},
+        #state{module=Module, sink=Sink} = State) ->
     % Brutally kill the manager but stay alive to restore settings.
-    Manager = whereis(Event),
+    Manager = whereis(Sink),
     unlink(Manager),
     exit(Manager, kill),
     erlang:send_after(KillerReinstallAfter, self(), reinstall_handler),
-    {noreply, State#state{config=[KillerHWM, KillerReinstallAfter]}};
+    {noreply, State};
 handle_info({gen_event_EXIT, Module, Reason}, #state{module=Module,
         config=Config, sink=Sink} = State) ->
     case lager:log(error, self(), "Lager event handler ~p exited with reason ~s",

src/lager_manager_killer.erl

@@ -7,38 +7,38 @@
 -include("lager.hrl").
 
 -record(state, {
-  killer_hwm :: non_neg_integer(),
-  killer_reinstall_after :: non_neg_integer()
-}).
+          killer_hwm :: non_neg_integer(),
+          killer_reinstall_after :: non_neg_integer()
+         }).
 
 init([KillerHWM, KillerReinstallAfter]) ->
-  {ok, #state{killer_hwm=KillerHWM, killer_reinstall_after=KillerReinstallAfter}}.
+    {ok, #state{killer_hwm=KillerHWM, killer_reinstall_after=KillerReinstallAfter}}.
 
 handle_call(get_loglevel, State) ->
-  {ok, {mask, ?LOG_NONE}, State};
+    {ok, {mask, ?LOG_NONE}, State};
 handle_call({set_loglevel, _Level}, State) ->
-  {ok, ok, State};
+    {ok, ok, State};
 handle_call(get_settings, State = #state{killer_hwm=KillerHWM, killer_reinstall_after=KillerReinstallAfter}) ->
-  {ok, [KillerHWM, KillerReinstallAfter], State};
+    {ok, [KillerHWM, KillerReinstallAfter], State};
 handle_call(_Request, State) ->
-  {ok, ok, State}.
+    {ok, ok, State}.
 
 handle_event({log, _Message}, State = #state{killer_hwm=KillerHWM, killer_reinstall_after=KillerReinstallAfter}) ->
-  {message_queue_len, Len} = process_info(self(), message_queue_len),
-  case Len > KillerHWM of
-    true ->
-      exit({kill_me, [KillerHWM, KillerReinstallAfter]});
-    _ ->
-      {ok, State}
-  end;
+    {message_queue_len, Len} = process_info(self(), message_queue_len),
+    case Len > KillerHWM of
+        true ->
+            exit({kill_me, [KillerHWM, KillerReinstallAfter]});
+        _ ->
+            {ok, State}
+    end;
 handle_event(_Event, State) ->
-  {ok, State}.
+    {ok, State}.
 
 handle_info(_Info, State) ->
-  {ok, State}.
+    {ok, State}.
 
 terminate(_Reason, _State) ->
-  ok.
+    ok.
 
 code_change(_OldVsn, State, _Extra) ->
-  {ok, State}.
+    {ok, State}.

test/lager_manager_killer_test.erl

@@ -12,7 +12,7 @@ overload_test() ->
   Delay = 1000, % sleep 1 sec on every log
   KillerHWM = 10, % kill the manager if there are more than 10 pending logs
   KillerReinstallAfter = 1000, % reinstall killer after 1 sec
-  application:set_env(lager, handlers, [{lager_slow_backend, Delay}]),
+  application:set_env(lager, handlers, [{lager_slow_backend, [{delay, Delay}]}]),
   application:set_env(lager, async_threshold, undefined),
   application:set_env(lager, killer_hwm, KillerHWM),
   application:set_env(lager, killer_reinstall_after, KillerReinstallAfter),
@@ -52,4 +52,4 @@ ensure_started(App) ->
       ensure_started(App)
   end.
 
--endif.
+-endif.

test/lager_slow_backend.erl

@@ -10,7 +10,7 @@
   delay :: non_neg_integer()
 }).
 
-init(Delay) ->
+init([{delay, Delay}]) ->
   {ok, #state{delay=Delay}}.
 
 handle_call(get_loglevel, State) ->