diff --git a/src/lager.erl b/src/lager.erl
index 475f35a..f77c822 100644
--- a/src/lager.erl
+++ b/src/lager.erl
@@ -294,7 +294,10 @@ stop_trace_int({_Filter, _Level, Backend} = Trace, Sink) ->
             %% check no other traces point here
             case lists:keyfind(Backend, 3, NewTraces) of
                 false ->
-                    gen_event:delete_handler(Sink, Backend, []);
+                    gen_event:delete_handler(Sink, Backend, []),
+                    lager_config:global_set(handlers,
+                                            lists:keydelete(Backend, 1,
+                                                            lager_config:global_get(handlers)));
                 _ ->
                     ok
             end;