Try to preserve internal log messages generated before lager boots

If backends have a failure or something, don't discard the messages,
wait half a second and then try sending them again.

include/lager.hrl

@@ -54,12 +54,32 @@
 -define(SHOULD_LOG(Level),
     lager_util:level_to_num(Level) =< lager_mochiglobal:get(loglevel, ?LOG_NONE)).
 
+-define(NOTIFY(Level, Pid, Format, Args),
+    gen_event:notify(lager_event, {log, lager_util:level_to_num(Level),
+            lager_util:format_time(), [io_lib:format("[~p] ", [Level]),
+                io_lib:format("~p ", [Pid]), io_lib:format(Format, Args)]})).
+
+%% FOR INTERNAL USE ONLY
 %% internal non-blocking logging call
+%% there's some special handing for when we try to log (usually errors) while
+%% lager is still starting.
 -define(INT_LOG(Level, Format, Args),
-    case ?SHOULD_LOG(Level) of
-        true ->
-            gen_event:notify(lager_event, {log, lager_util:level_to_num(Level),
-                    lager_util:format_time(), [io_lib:format("[~p] ", [Level]), io_lib:format("~p ", [self()]), io_lib:format(Format, Args)]});
-        _ -> ok
-    end).
-
+    Self = self(),
+    %% do this in a spawn so we don't cause a deadlock calling gen_event:which_handlers
+    %% from a gen_event handler
+    spawn(fun() ->
+            case catch(gen_event:which_handlers(lager_event)) of
+                X when X == []; X == {'EXIT', noproc} ->
+                    %% there's no handlers yet or lager isn't running, try again
+                    %% in half a second.
+                    timer:sleep(500),
+                    ?NOTIFY(Level, Self, Format, Args);
+                _ ->
+                    case ?SHOULD_LOG(Level) of
+                        true ->
+                            ?NOTIFY(Level, Self, Format, Args);
+                        _ ->
+                            ok
+                    end
+            end
+    end)).

src/lager_app.erl

@@ -21,6 +21,7 @@
 -module(lager_app).
 
 -behaviour(application).
+-include("lager.hrl").
 
 -export([start/0,
          start/2,
@@ -30,6 +31,8 @@ start() ->
     application:start(lager).
 
 start(_StartType, _StartArgs) ->
+    %% until lager is completely started, allow all messages to go through
+    lager_mochiglobal:put(loglevel, ?DEBUG),
     {ok, Pid} = lager_sup:start_link(),
     Handlers = case application:get_env(lager, handlers) of
         undefined ->
@@ -42,6 +45,7 @@ start(_StartType, _StartArgs) ->
     [supervisor:start_child(lager_handler_watcher_sup, [lager_event, Module, Config]) ||
         {Module, Config} <- Handlers],
 
+    %% mask the messages we have no use for
     MinLog = lager:minimum_loglevel(lager:get_loglevels()),
     lager_mochiglobal:put(loglevel, MinLog),