From 81d4aea7d864cbf541b4ee495d9b37bf061f7026 Mon Sep 17 00:00:00 2001 From: Andrew Thompson Date: Sat, 23 Jul 2011 00:19:50 -0400 Subject: [PATCH] Finish implementing time based log rotation --- README.org | 57 ++++++++++++++++++++++++++++++++++++-- src/lager.app.src | 3 ++ src/lager_crash_log.erl | 30 ++++++++++++++------ src/lager_file_backend.erl | 41 +++++++++++++++++++++++---- src/lager_sup.erl | 14 +++++++++- src/lager_util.erl | 2 ++ 6 files changed, 129 insertions(+), 18 deletions(-) diff --git a/README.org b/README.org index b1571ff..16048d1 100644 --- a/README.org +++ b/README.org @@ -54,8 +54,8 @@ {handlers, [ {lager_console_backend, info}, {lager_file_backend, [ - {"error.log", error, 10485760, "", 5}, - {"console.log", info, 10485760, "", 5} + {"error.log", error, 10485760, "$D0", 5}, + {"console.log", info, 10485760, "$D0", 5} ]} ]} }. @@ -97,3 +97,56 @@ log messages, when no backend is consuming debug messages, are effectively free. A simple benchmark of doing 1 million debug log messages while the minimum threshold was above that takes less than half a second. + +* Internal log rotation + Lager can rotate its own logs or have it done via an external process. To + use internal rotation, use the last 3 values in the file backend's + configuration tuple. For example + +#+BEGIN_EXAMPLE + {"error.log", error, 10485760, "$D0", 5} +#+END_EXAMPLE + + This tells lager to log error and above messages to "error.log" and to + rotate the file at midnight or when it reaches 10mb, whichever comes first + and to keep 5 rotated logs, in addition to the current one. Setting the + count to 0 does not disable rotation, it instead rotates the file and keeps + no previous versions around. To disable rotation set the size to 0 and the + date to "". + + The "$D0" syntax is taken from the syntax newsyslog uses in newsyslog.conf. + The relevant extract follows: + +#+BEGIN_EXAMPLE + Day, week and month time format: The lead-in character + for day, week and month specification is a `$'-sign. + The particular format of day, week and month + specification is: [Dhh], [Ww[Dhh]] and [Mdd[Dhh]], + respectively. Optional time fields default to + midnight. The ranges for day and hour specifications + are: + + hh hours, range 0 ... 23 + w day of week, range 0 ... 6, 0 = Sunday + dd day of month, range 1 ... 31, or the + letter L or l to specify the last day of + the month. + + Some examples: + $D0 rotate every night at midnight + $D23 rotate every day at 23:00 hr + $W0D23 rotate every week on Sunday at 23:00 hr + $W5D16 rotate every week on Friday at 16:00 hr + $M1D0 rotate on the first day of every month at + midnight (i.e., the start of the day) + $M5D6 rotate on every 5th day of the month at + 6:00 hr +#+END_EXAMPLE + + To configure the crash log rotation, the following application variables are + used: + - crash_log_size + - crash_log_date + - crash_log_count + + See the .app.src file for further details. diff --git a/src/lager.app.src b/src/lager.app.src index 41525ae..1a4c8b3 100644 --- a/src/lager.app.src +++ b/src/lager.app.src @@ -27,6 +27,9 @@ %% Maximum size of the crash log in bytes, before its rotated, set %% to 0 to disable rotation - default is 0 {crash_log_size, 10485760}, + %% What time to rotate the crash log - default is no time + %% rotation. See the README for a description of this format. + {crash_log_date, "$D0"}, %% Number of rotated crash logs to keep, 0 means keep only the %% current one - default is 0 {crash_log_count, 5}, diff --git a/src/lager_crash_log.erl b/src/lager_crash_log.erl index c2821af..9023632 100644 --- a/src/lager_crash_log.erl +++ b/src/lager_crash_log.erl @@ -22,9 +22,10 @@ %% %% The `crash_log_msg_size' application var is used to specify the maximum %% size of any message to be logged. `crash_log_size' is used to specify the -%% maximum size of the crash log before it will be rotated (0 will disable) -%% and to control the number of rotated files to be retained, use -%% `crash_log_count'. +%% maximum size of the crash log before it will be rotated (0 will disable). +%% Time based rotation is configurable via `crash_log_date', the syntax is +%% documented in the README. To control the number of rotated files to be +%% retained, use `crash_log_count'. -module(lager_crash_log). @@ -44,26 +45,28 @@ inode, fmtmaxbytes, size, + date, count, flap=false }). %% @private -start_link(Filename, MaxBytes, Size, _Date, Count) -> +start_link(Filename, MaxBytes, Size, Date, Count) -> gen_server:start_link({local, ?MODULE}, ?MODULE, [Filename, MaxBytes, - Size, Count], []). + Size, Date, Count], []). %% @private -start(Filename, MaxBytes, Size, _Date, Count) -> +start(Filename, MaxBytes, Size, Date, Count) -> gen_server:start({local, ?MODULE}, ?MODULE, [Filename, MaxBytes, Size, - Count], []). + Date, Count], []). %% @private -init([Filename, MaxBytes, Size, Count]) -> +init([Filename, MaxBytes, Size, Date, Count]) -> case lager_util:open_logfile(Filename, false) of {ok, {FD, Inode, _}} -> + schedule_rotation(Date), {ok, #state{name=Filename, fd=FD, inode=Inode, - fmtmaxbytes=MaxBytes, size=Size, count=Count}}; + fmtmaxbytes=MaxBytes, size=Size, count=Count, date=Date}}; Error -> Error end. @@ -120,6 +123,10 @@ handle_cast(_Request, State) -> {noreply, State}. %% @private +handle_info(rotate, #state{name=Name, count=Count, date=Date} = State) -> + lager_util:rotate_logfile(Name, Count), + schedule_rotation(Date), + {noreply, State}; handle_info(_Info, State) -> {noreply, State}. @@ -131,6 +138,11 @@ terminate(_Reason, _State) -> code_change(_OldVsn, State, _Extra) -> {ok, State}. +schedule_rotation(undefined) -> + make_ref(); +schedule_rotation(Date) -> + erlang:send_after(lager_util:calculate_next_rotation(Date) * 1000, self(), rotate). + %% ===== Begin code lifted from riak_err ===== -spec limited_fmt(string(), list(), integer()) -> iolist(). diff --git a/src/lager_file_backend.erl b/src/lager_file_backend.erl index b7f1159..e2b0c58 100644 --- a/src/lager_file_backend.erl +++ b/src/lager_file_backend.erl @@ -23,7 +23,8 @@ %% rotation and will re-open handles to files if the inode changes. It will %% also rotate the files itself if the size of the file exceeds the %% `RotationSize' and keep `RotationCount' rotated files. `RotationDate' is -%% currently ignored. +%% an alternate rotation trigger, based on time. See the README for +%% documentation. -module(lager_file_backend). @@ -57,6 +58,7 @@ -spec init([{string(), lager:log_level()},...]) -> {ok, #state{}}. init(LogFiles) -> Files = [begin + schedule_rotation(Name, Date), case lager_util:open_logfile(Name, true) of {ok, {FD, Inode, _}} -> #file{name=Name, level=lager_util:level_to_num(Level), fd=FD, @@ -109,6 +111,17 @@ handle_event(_Event, State) -> {ok, State}. %% @private +handle_info({rotate, File}, #state{files=Files} = State) -> + case lists:keyfind(File, #file.name, Files) of + false -> + %% no such file exists + ?INT_LOG(warning, "Asked to rotate non-existant file ~p", [File]), + {ok, State}; + #file{name=Name, date=Date, count=Count} -> + lager_util:rotate_logfile(Name, Count), + schedule_rotation(Name, Date), + {ok, State} + end; handle_info(_Info, State) -> {ok, State}. @@ -167,27 +180,43 @@ validate_logfiles([{Name, Level, Size, Date, Count}|T]) -> true -> case (is_integer(Count) andalso Count >= 0) of true -> - [{Name, Level, Size, Date, - Count}|validate_logfiles(T)]; + case lager_util:parse_rotation_date_spec(Date) of + {ok, Spec} -> + [{Name, Level, Size, Spec, + Count}|validate_logfiles(T)]; + {error, _} when Date == "" -> + %% blank ones are fine. + [{Name, Level, Size, undefined, + Count}|validate_logfiles(T)]; + {error, _} -> + ?INT_LOG(error, "Invalid rotation date of ~p for ~s.", + [Date, Name]), + validate_logfiles(T) + end; _ -> ?INT_LOG(error, "Invalid rotation count of ~p for ~s.", - [Name, Count]), + [Count, Name]), validate_logfiles(T) end; _ -> ?INT_LOG(error, "Invalid rotation size of ~p for ~s.", - [Name, Size]), + [Size, Name]), validate_logfiles(T) end; _ -> ?INT_LOG(error, "Invalid log level of ~p for ~s.", - [Name, Level]), + [Level, Name]), validate_logfiles(T) end; validate_logfiles([H|T]) -> ?INT_LOG(error, "Invalid logfile config ~p.", [H]), validate_logfiles(T). +schedule_rotation(_, undefined) -> + make_ref(); +schedule_rotation(Name, Date) -> + erlang:send_after(lager_util:calculate_next_rotation(Date) * 1000, self(), {rotate, Name}). + -ifdef(TEST). get_loglevel_test() -> diff --git a/src/lager_sup.erl b/src/lager_sup.erl index 2f64cae..d780199 100644 --- a/src/lager_sup.erl +++ b/src/lager_sup.erl @@ -53,9 +53,21 @@ init([]) -> {ok, Val2} when is_integer(Val2) -> Val2; _ -> 0 end, + RotationDate = case application:get_env(lager, crash_log_date) of + {ok, Val3} -> + case lager_util:parse_rotation_date_spec(Val3) of + {ok, Spec} -> Spec; + {error, _} when Val3 == "" -> undefined; %% blank is ok + {error, _} -> + error_logger:error_msg("Invalid date spec for " + "crash log ~p~n", [Val3]), + undefined + end; + _ -> undefined + end, [{lager_crash_log, {lager_crash_log, start_link, [File, MaxBytes, - RotationSize, "", RotationCount]}, + RotationSize, RotationDate, RotationCount]}, permanent, 5000, worker, [lager_crash_log]}]; _ -> [] diff --git a/src/lager_util.erl b/src/lager_util.erl index 1e55a79..304af78 100644 --- a/src/lager_util.erl +++ b/src/lager_util.erl @@ -318,6 +318,8 @@ rotation_calculation_test() -> ?assertMatch({{2000, 2, 3}, {16, 0, 0}}, calculate_next_rotation([{day, 4}, {hour, 16}], {{2000, 1, 29}, {17, 34, 43}})), ?assertMatch({{2000, 1, 7}, {16, 0, 0}}, calculate_next_rotation([{day, 5}, {hour, 16}], {{2000, 1, 3}, {17, 34, 43}})), + + ?assertMatch({{2000, 1, 3}, {16, 0, 0}}, calculate_next_rotation([{day, 1}, {hour, 16}], {{1999, 12, 28}, {17, 34, 43}})), ok. -endif.