From 4aac990e707af9909c240d7ec4879ae06abf99ef Mon Sep 17 00:00:00 2001 From: SisMaker <1713699517@qq.com> Date: Fri, 28 May 2021 00:33:42 +0800 Subject: [PATCH] =?UTF-8?q?ft:=20=E4=BB=A3=E7=A0=81=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 6 +-- include/eTpf.hrl | 54 ++++++++++++++----------- src/eTpf.erl | 49 +++++++++++------------ src/tracer/tpTracerConsole.erl | 2 +- src/tracer/tpTracerFile.erl | 62 ++++++++++++++++++++++------- src/tracer/tpTracerPool.erl | 3 -- src/tracer/tpTracerSocket.erl | 3 +- src/utils/tpTermCut.erl | 73 +++++++++++++++++++++------------- test/lg_SUITE.erl | 6 +-- 9 files changed, 156 insertions(+), 102 deletions(-) diff --git a/README.md b/README.md index c51d7a9..527a22b 100644 --- a/README.md +++ b/README.md @@ -235,13 +235,13 @@ end. 对于长时间运行的会话，eTpf可以旋转跟踪文件。这是一项有助于避免磁盘空间用完的功能，并且不用于保留较小的文件（eTpf可以处理非常大的文件就可以了）。 -eTpf:trace/3,4可以提供一个映射，而不是将文件名前缀作为第三个参数传递 给。当前有三个选项，包括filename_prefix。其他选项是最大文件大小（以字节为单位）max_size，以及将在文件中每LZ4帧存储的事件数 -events_per_frame。这两个选项使您可以控制文件写入或旋转的频率。 +eTpf:trace/3,4可以提供一个映射，而不是将文件名前缀作为第三个参数传递 给。当前有三个选项，包括filename_prefix。其他选项是最大文件大小（以字节为单位）fMaxSize，以及将在文件中每LZ4帧存储的事件数 +fEventFrame。这两个选项使您可以控制文件写入或旋转的频率。 以下示例将文件大小限制为100MB： ``` -1> eTpf:trace('_', lg_file_tracer, #{filename_prefix => "traces.lz4", max_size => 100000000}, #{mode => utils, running =>true}). +1> eTpf:trace('_', lg_file_tracer, #{fBaseName => "traces.lz4", fMaxSize => 100000000}, #{mode => utils, running =>true}). ``` 在测试此功能期间，目前实施的轮换似乎很昂贵，因此您应注意不要设置太低的值。 diff --git a/include/eTpf.hrl b/include/eTpf.hrl index c8be261..69955a1 100644 --- a/include/eTpf.hrl +++ b/include/eTpf.hrl @@ -2,14 +2,18 @@ -define(eTpfHole, '$eTpfHole'). %% 消息截取默认配置项 --define(tcmIsCut, false). --define(tcmDepth, 3). --define(tcmListSize, 32). --define(tcmMapSize, 32). --define(tcmTupleSize, 32). --define(tcmBinSize, 128). --define(tcmBitSize, ?tcmBinSize * 8). --define(tcmNestStruct, 6). +-define(defTcmMap, #{ + tcmIsCut => false + , tcmDepth => 5 + , tcmListSize => 32 + , tcmMapSize => 32 + , tcmTupleSize => 32 + , tcmBinSize => 128 + , tcmBitSize => 128 * 8 + , tcmNestStruct => 6 +}). + +-define(defPoolId, defPoolId). %% 类型声明 -export_type([input/0, userInput/0, traceOpts/0, tracerOpts/0]). @@ -22,30 +26,32 @@ -type userInput() :: pattern() | input(). -type traceOpts() :: #{ -mode => trace | profile -, poolId => any() -, poolSize => pos_integer() -, send => boolean() -, running => boolean() + mode => trace | profile + , poolId => any() + , poolSize => pos_integer() + , send => boolean() + , running => boolean() + , process_dump => boolean() %% 必需的trace选项 为了生成火焰图，我们目前在跟踪时需要使用一个附加选项。此选项将导致将堆栈跟踪信息添加到调用事件。选项为`process_dump`, 并且必须将其设置为`true`。 }. -type tracerOpts() :: #{ %% tracer for socket - port => pos_integer() + port => pos_integer() %% tracer for file - ,filename_prefix => string() - , max_size => pos_integer() - , events_per_frame => pos_integer() + , fBaseName => string() + , fMaxSize => pos_integer() + , fEventFrame => pos_integer() %% tracer for console %% trace msg term cut cfg - ,tcmIsCut => boolean() %% 是否cut trace msg term - ,tcmDepth => boolean() %% term cut时保留的最大深度 - ,tcmListSize => boolean() %% List term cut时保留的最大数量 - ,tcmMapSize => boolean() %% Map term cut时保留的最大数量 - ,tcmTupleSize => boolean() %% Tuple term cut时保留的最大数量 - ,tcmBitSize => boolean() %% Bits term cut时保留的最大字节数 - ,tcmNestStruct => boolean() %% 复合结构 term cut时保留的最大数量 + , tcmIsCut => boolean() %% 是否cut trace msg term + , tcmDepth => pos_integer() %% term cut时保留的最大深度 + , tcmListSize => pos_integer() %% List term cut时保留的最大数量 + , tcmMapSize => pos_integer() %% Map term cut时保留的最大数量 + , tcmTupleSize => pos_integer() %% Tuple term cut时保留的最大数量 + , tcmBinSize => pos_integer() %% Bin term cut时保留的最大字节数 + , tcmBitSize => pos_integer() %% Bits term cut时保留的最大bit数 + , tcmNestStruct => pos_integer() %% 复合结构 term cut时保留的最大数量 }. \ No newline at end of file diff --git a/src/eTpf.erl b/src/eTpf.erl index cf24056..6e34319 100644 --- a/src/eTpf.erl +++ b/src/eTpf.erl @@ -27,15 +27,14 @@ trace(Input, TracerMod, TracerOpts) -> -spec trace(userInput(), module(), tracerOpts(), traceOpts()) -> ok. trace(Input, TracerMod, TracerOpts, TraceOpts) when is_list(Input) -> - do_trace(Input, TracerMod, TracerOpts, TraceOpts); -trace(Input, TracerMod, TracerOpts, TraceOpts) -> - trace([Input], TracerMod, TracerOpts, TraceOpts). + InputList = case is_list(Input) of true -> Input; _ -> [Input] end, + do_trace(InputList, TracerMod, TracerOpts, TraceOpts). -do_trace(Input0, TracerMod, TracerOpts, TraceOpts) -> +do_trace(InputList, TracerMod, TracerOpts, TraceOpts) -> _ = application:ensure_all_started(eTpf), %% Start the pool of tracer processes. - PoolID = maps:get(poolId, TraceOpts, default), + PoolID = maps:get(poolId, TraceOpts, ?defPoolId), PoolSize = maps:get(poolSize, TraceOpts, erlang:system_info(schedulers)), true = PoolSize > 0, {ok, PoolPid} = supervisor:start_child(eTpf_sup, #{ @@ -47,39 +46,39 @@ do_trace(Input0, TracerMod, TracerOpts, TraceOpts) -> Tracers = tpTracerPool:tracers(PoolPid), TracersMap = maps:from_list(lists:zip(lists:seq(0, length(Tracers) - 1), Tracers)), Mode = maps:get(mode, TraceOpts, trace), - Input1 = flatten(Input0, []), - Input2 = ensure_pattern(Input1), - Input = ensure_scope(Input2), - trace_input(Input, #{mode => Mode, tracers => TracersMap}, TraceOpts), + Tem0InputList = flattenInput(InputList, []), + Tem1InputList = ensurePattern(Tem0InputList), + LastInputList = ensureScope(Tem1InputList), + traceInput(LastInputList, #{mode => Mode, tracers => TracersMap}, TraceOpts), ok. -flatten([], Acc) -> +flattenInput([], Acc) -> lists:flatten(Acc); -flatten([{callback, Mod, Fun} | Tail], Acc) when is_atom(Mod), is_atom(Fun) -> - Input = flatten(Mod:Fun(), []), - flatten(Tail, [Input | Acc]); -flatten([{app, App} | Tail], Acc) when is_atom(App) -> +flattenInput([{callback, Mod, Fun} | Tail], Acc) when is_atom(Mod), is_atom(Fun) -> + Input = flattenInput(Mod:Fun(), []), + flattenInput(Tail, [Input | Acc]); +flattenInput([{app, App} | Tail], Acc) when is_atom(App) -> _ = application:load(App), {ok, Mods} = application:get_key(App, modules), - flatten(Tail, [Mods | Acc]); -flatten([Input | Tail], Acc) -> - flatten(Tail, [Input | Acc]). + flattenInput(Tail, [Mods | Acc]); +flattenInput([Input | Tail], Acc) -> + flattenInput(Tail, [Input | Acc]). -ensure_pattern(Input) -> +ensurePattern(Input) -> case [S || S = {scope, _} <- Input] of Input -> ['_' | Input]; _ -> Input end. -ensure_scope(Input) -> +ensureScope(Input) -> case [S || S = {scope, _} <- Input] of [] -> [{scope, [processes]} | Input]; _ -> Input end. -trace_input([], _, _) -> +traceInput([], _, _) -> ok; -trace_input([{scope, Scope} | Tail], TracerState, Opts) -> +traceInput([{scope, Scope} | Tail], TracerState, Opts) -> %% We currently enable the following trace flags: %% - call: function calls %% - procs: process exit events; plus others we ignore @@ -96,8 +95,8 @@ trace_input([{scope, Scope} | Tail], TracerState, Opts) -> erlang:trace(PidPortSpec, true, [call, procs, timestamp, arity, return_to, set_on_spawn, {tracer, tpTracerNif, TracerState} | ExtraFlags]) || PidPortSpec <- Scope ], - trace_input(Tail, TracerState, Opts); -trace_input([Mod | Tail], TracerState, Opts) when is_atom(Mod) -> + traceInput(Tail, TracerState, Opts); +traceInput([Mod | Tail], TracerState, Opts) when is_atom(Mod) -> MatchSpec = case Opts of #{process_dump := true} -> @@ -108,10 +107,10 @@ trace_input([Mod | Tail], TracerState, Opts) when is_atom(Mod) -> %% The module must be loaded before we attempt to trace it. _ = code:ensure_loaded(Mod), _ = erlang:trace_pattern({Mod, '_', '_'}, MatchSpec, [local]), - trace_input(Tail, TracerState, Opts). + traceInput(Tail, TracerState, Opts). stop() -> - stop(default). + stop(?defPoolId). %% @todo Confirm that we don't need to stop tracing, %% that just terminating the tracers is enough. The diff --git a/src/tracer/tpTracerConsole.erl b/src/tracer/tpTracerConsole.erl index 5bcb5a1..120f5d8 100644 --- a/src/tracer/tpTracerConsole.erl +++ b/src/tracer/tpTracerConsole.erl @@ -8,7 +8,7 @@ -export([system_terminate/4]). -export([system_code_change/4]). -start_link(_Nth, _Opts) -> +start_link(_Nth, _TracerOpts) -> Pid = proc_lib:spawn_link(?MODULE, init, [self()]), {ok, Pid}. diff --git a/src/tracer/tpTracerFile.erl b/src/tracer/tpTracerFile.erl index 399e446..ac982e4 100644 --- a/src/tracer/tpTracerFile.erl +++ b/src/tracer/tpTracerFile.erl @@ -1,4 +1,5 @@ -module(tpTracerFile). +-include("eTpf.hrl"). -export([ start_link/2 @@ -13,39 +14,70 @@ ]). -record(state, { - parent :: pid(), - fileName :: file:filename_all(), - size = 0 :: non_neg_integer(), - maxSize :: infinity | non_neg_integer(), - ioDevice :: file:io_device(), - eventsPerFrame :: pos_integer(), - eventsThisFrame = 0 :: non_neg_integer(), - buffer = <<>> :: binary() + parent :: pid() + , fBaseName :: file:filename_all() + , size = 0 :: non_neg_integer() + , fMaxSize :: infinity | non_neg_integer() + , ioDevice :: file:io_device() + , fEventFrame :: pos_integer() + , eventsThisFrame = 0 :: non_neg_integer() + , buffer = <<>> :: binary() + + , tcmIsCut = false + , tcmDepth = 5 + , tcmListSize = 32 + , tcmMapSize = 32 + , tcmTupleSize = 32 + , tcmBinSize = 128 + , tcmBitSize = 128 * 8 + , tcmNestStruct = 6 }). start_link(Nth, Prefix) when is_list(Prefix) -> - start_link(Nth, #{filename_prefix => Prefix}); + start_link(Nth, #{fBaseName => Prefix}); start_link(Nth, Opts) when is_map(Opts) -> Pid = proc_lib:spawn_link(?MODULE, init, [self(), Nth, Opts]), {ok, Pid}. -init(Parent, Nth, Opts) -> +init(Parent, Nth, TracerOpts) -> process_flag(message_queue_data, off_heap), process_flag(trap_exit, true), %% No need to close the file, it'll be closed when the process exits. - Filename = filename:flatten([maps:get(filename_prefix, Opts, "traces.lz4"), ".", integer_to_list(Nth)]), + Filename = filename:flatten([maps:get(fBaseName, TracerOpts, "traces.lz4"), ".", integer_to_list(Nth)]), {ok, IoDevice} = file:open(Filename, [write, raw]), - State = #state{parent = Parent, fileName = Filename, ioDevice = IoDevice, maxSize = maps:get(max_size, Opts, infinity), eventsPerFrame = maps:get(events_per_frame, Opts, 100000)}, + + TcmIsCut = maps:get(tcmIsCut, TracerOpts, maps:get(tcmDepth, ?defTcmMap)), + case TcmIsCut of + true -> + erlang:put(tcmDepth, maps:get(tcmDepth, TracerOpts, maps:get(tcmDepth, ?defTcmMap))), + erlang:put(tcmListSize, maps:get(tcmListSize, TracerOpts, maps:get(tcmListSize, ?defTcmMap))), + erlang:put(tcmMapSize, maps:get(tcmMapSize, TracerOpts, maps:get(tcmMapSize, ?defTcmMap))), + erlang:put(tcmTupleSize, maps:get(tcmTupleSize, TracerOpts, maps:get(tcmTupleSize, ?defTcmMap))), + erlang:put(tcmBinSize, maps:get(tcmBinSize, TracerOpts, maps:get(tcmBinSize, ?defTcmMap))), + erlang:put(tcmBitSize, maps:get(tcmBitSize, TracerOpts, maps:get(tcmBitSize, ?defTcmMap))), + erlang:put(tcmNestStruct, maps:get(tcmNestStruct, TracerOpts, maps:get(tcmNestStruct, ?defTcmMap))); + _ -> + ignore + end, + + State = #state{ + parent = Parent + , fBaseName = Filename + , ioDevice = IoDevice + , fMaxSize = maps:get(fMaxSize, TracerOpts, infinity) + , fEventFrame = maps:get(fEventFrame, TracerOpts, 100000) + , tcmIsCut = TcmIsCut + }, loop(State). -loop(State = #state{parent = Parent, size = Size, ioDevice = IoDevice, eventsPerFrame = MaxEvents, eventsThisFrame = NumEvents0, buffer = Buffer0}) -> +loop(State = #state{parent = Parent, size = Size, ioDevice = IoDevice, fEventFrame = MaxEvents, eventsThisFrame = NumEvents0, buffer = Buffer0, tcmIsCut = TcmIsCut}) -> receive {system, From, Request} -> sys:handle_system_msg(Request, From, Parent, ?MODULE, [], State); {'EXIT', Parent, Reason} -> terminate(Reason, State); RMsg -> - Msg = tpTermCut:cut(RMsg), + Msg = case TcmIsCut of true -> tpTermCut:cut(RMsg); _ -> RMsg end, Bin = term_to_binary(Msg), BinSize = byte_size(Bin), Buffer = <>, @@ -62,7 +94,7 @@ loop(State = #state{parent = Parent, size = Size, ioDevice = IoDevice, eventsPer end. %%IMY-todo 干掉这个函数 同时对于旋转的文件名 使用时间 年月日时分 -maybe_rotate(State = #state{fileName = Filename, size = Size, maxSize = MaxSize, +maybe_rotate(State = #state{fBaseName = Filename, size = Size, fMaxSize = MaxSize, ioDevice = OldIoDevice}) when Size > MaxSize -> ok = file:close(OldIoDevice), ok = file:rename(Filename, Filename ++ ".bak"), diff --git a/src/tracer/tpTracerPool.erl b/src/tracer/tpTracerPool.erl index df7009f..f47cb99 100644 --- a/src/tracer/tpTracerPool.erl +++ b/src/tracer/tpTracerPool.erl @@ -7,7 +7,6 @@ , tracers/1 ]). - start_link(NumTracers, TracerMod, Opts) -> supervisor:start_link(?MODULE, [NumTracers, TracerMod, Opts]). @@ -22,7 +21,5 @@ init([NumTracers, TracerMod, Opts]) -> SupFlags = #{strategy => one_for_all, intensity => 5, period => 10}, {ok, {SupFlags, ChildSpecs}}. - - tracers(PoolPid) -> [Pid || {_, Pid, _, _} <- supervisor:which_children(PoolPid)]. diff --git a/src/tracer/tpTracerSocket.erl b/src/tracer/tpTracerSocket.erl index 8c0b639..78519c1 100644 --- a/src/tracer/tpTracerSocket.erl +++ b/src/tracer/tpTracerSocket.erl @@ -18,7 +18,8 @@ timerRef :: reference() | undefined }). -start_link(Nth, BasePort) -> +start_link(Nth, TracerOpts) -> + BasePort = maps:get(port, TracerOpts), Pid = proc_lib:spawn_link(?MODULE, init, [self(), BasePort + Nth - 1]), {ok, Pid}. diff --git a/src/utils/tpTermCut.erl b/src/utils/tpTermCut.erl index b51cf2b..2009633 100644 --- a/src/utils/tpTermCut.erl +++ b/src/utils/tpTermCut.erl @@ -6,50 +6,69 @@ -export([cut/1]). -export([cut/2]). - +getTcmValue(Key) -> + case get(Key) of + undefined -> + maps:get(Key, ?defTcmMap); + Value -> + Value + end. cut(Term) -> cut(Term, 1). -cut(_, Depth) when Depth > ?tcmDepth -> - '$truncated11'; -cut(Bits, _) when is_bitstring(Bits), bit_size(Bits) > ?tcmBitSize -> - <> = Bits, - <>; -cut(Term, Depth) when is_list(Term) -> - cutList(Term, Depth, 0, ?tcmListSize, 0, []); -cut(Term, Depth) when is_map(Term), Depth =:= ?tcmDepth -> - #{'$truncated' => '$truncated'}; -cut(Term, Depth) when is_map(Term) -> - maps:from_list(cutMap(maps_to_list(Term, ?tcmMapSize), Depth, 0)); -cut(Term, Depth) when is_tuple(Term), Depth =:= ?tcmDepth -> - {'$truncated'}; -cut(Term, Depth) when is_tuple(Term) -> - list_to_tuple(cutList(tuple_to_list(Term), Depth, 0, ?tcmTupleSize, 0, [])); -cut(Term, _) -> - Term. +cut(Term, Depth) -> + TcmDepth = getTcmValue(tcmDepth), + case Depth > TcmDepth of + true -> + '$truncated11'; + _ -> + if + is_bitstring(Term) -> + case bit_size(Term) > getTcmValue(tcmBitSize) of + true -> + TcmBinSize = getTcmValue(tcmBinSize), + <> = Term, + <>; + _ -> + Term + end; + is_list(Term) -> + cutList(Term, Depth, 0, getTcmValue(tcmListSize), getTcmValue(tcmNestStruct), 0, []); + is_map(Term), Depth =:= TcmDepth -> + #{'$truncated' => '$truncated'}; + is_map(Term) -> + maps:from_list(cutMap(maps_to_list(Term, getTcmValue(tcmMapSize)), Depth, getTcmValue(tcmNestStruct), 0)); + is_tuple(Term), Depth =:= TcmDepth -> + {'$truncated'}; + is_tuple(Term) -> + list_to_tuple(cutList(tuple_to_list(Term), Depth, 0, getTcmValue(tcmTupleSize), getTcmValue(tcmNestStruct), 0, [])); + true -> + Term + end + end. -cutList([], _, _, _, _, Acc) -> +cutList([], _, _, _, _, _, Acc) -> lists:reverse(Acc); -cutList([Term | Tail], Depth, Len, MaxLen, NumStructs, Acc) -> - case Len >= MaxLen orelse NumStructs >= ?tcmNestStruct of +cutList([Term | Tail], Depth, Len, TcmTupleSize, TcmNestStruct, NumStructs, Acc) -> + case Len >= TcmTupleSize orelse NumStructs >= TcmNestStruct of true -> lists:reverse(['$truncated33' | Acc]); _ -> %% if List was a cons, Tail can be anything io:format("IMY********************1111 ~p~n", [{Len, NumStructs, Term}]), - cutList(Tail, Depth, Len + 1, MaxLen, NumStructs + isStruct(Term), [cut(Term, Depth + 1) | Acc]) + cutList(Tail, Depth, Len + 1, TcmTupleSize, TcmNestStruct, NumStructs + isStruct(Term), [cut(Term, Depth + 1) | Acc]) end; -cutList(Term, Depth, _, _, _, Acc) -> %% if List was a cons [[[a,a,a|b],1,2,3]|bbbb] +cutList(Term, Depth, _, _, _, _, Acc) -> %% if List was a cons [[[a,a,a|b],1,2,3]|bbbb] lists:reverse(Acc) ++ cut(Term, Depth + 1). -cutMap([], _, _) -> +cutMap([], _, _, _) -> []; -cutMap(_, _, NumStructs) when NumStructs > ?tcmNestStruct -> +cutMap(_, _, TcmNestStruct, NumStructs) when NumStructs > TcmNestStruct -> [{'$truncated', '$truncated'}]; -cutMap([{Key, Value} | Tail], Depth, NumStructs) -> +cutMap([{Key, Value} | Tail], Depth, TcmNestStruct, NumStructs) -> AddStruct = isStruct(Key) + isStruct(Value), - [{cut(Key, Depth + 1), cut(Value, Depth + 1)} | cutMap(Tail, Depth, NumStructs + AddStruct)]. + [{cut(Key, Depth + 1), cut(Value, Depth + 1)} | cutMap(Tail, Depth, TcmNestStruct, NumStructs + AddStruct)]. isStruct(Term) -> case is_list(Term) orelse is_map(Term) orelse is_tuple(Term) of diff --git a/test/lg_SUITE.erl b/test/lg_SUITE.erl index a7afa58..703ee1a 100644 --- a/test/lg_SUITE.erl +++ b/test/lg_SUITE.erl @@ -135,9 +135,9 @@ file_tracer_rotation(Config) -> doc("Save events to files on disk; rotate the files if they get too big."), Prefix = config(priv_dir, Config) ++ "/file_tracer.lz4", eTpf:trace(lists, tpTracerFile, #{ - filename_prefix => Prefix, - max_size => 100, %% Intentionally low. - events_per_frame => 10 %% Needed to trigger the rotation, default is too high. + fBaseName => Prefix, + fMaxSize => 100, %% Intentionally low. + fEventFrame => 10 %% Needed to trigger the rotation, default is too high. }), lists:seq(1, 1000), eTpf:stop(),