From 10ee7888330a7bcb95674a6777fcf8a3d9adad2c Mon Sep 17 00:00:00 2001 From: SisMaker <1713699517@qq.com> Date: Thu, 25 Feb 2021 12:43:43 +0800 Subject: [PATCH 1/4] =?UTF-8?q?=E4=BB=A3=E7=A0=81=E6=95=B4=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/srvNodeMgr/utVMInfo.erl | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/src/srvNodeMgr/utVMInfo.erl b/src/srvNodeMgr/utVMInfo.erl index 9b890be..d6f80e5 100644 --- a/src/srvNodeMgr/utVMInfo.erl +++ b/src/srvNodeMgr/utVMInfo.erl @@ -464,22 +464,35 @@ get_processes() -> get_process_info_and_large_than_value(memory, 0)). memInfoInit(CurModule, CurLine) -> - erlang:put(?pdMemInfo, {CurModule, CurLine, erlang:system_time(nanosecond), recon:info(self(), memory_used)}). + erlang:put(?pdMemInfo, {CurModule, CurLine, erlang:system_time(nanosecond), recon:info(self(), memory_used), erlang:memory()}). memInfoPrint(CurModule, CurLine, Threshold) -> case erlang:get(?pdMemInfo) of undefined -> - erlang:put(?pdMemInfo, {CurModule, CurLine, erlang:system_time(nanosecond), recon:info(self(), memory_used)}); - {OldModule, OldLine, OldTime, OldMemInfo} -> + erlang:put(?pdMemInfo, {CurModule, CurLine, erlang:system_time(nanosecond), recon:info(self(), memory_used), erlang:memory()}); + {OldModule, OldLine, OldTime, OldMemInfo, OldSumInfo} -> CurMemInfo = recon:info(self(), memory_used), CurTime = erlang:system_time(nanosecond), - erlang:put(?pdMemInfo, {CurModule, CurLine, CurTime, recon:info(self(), memory_used)}), - OldUsed = element(2, lists:nth(1, element(2, OldMemInfo))), - CurUsed = element(2, lists:nth(1, element(2, CurMemInfo))), - Sub = CurUsed - OldUsed, - case erlang:abs(Sub) >= Threshold of + CurSumInfo = erlang:memory(), + erlang:put(?pdMemInfo, {CurModule, CurLine, CurTime, CurMemInfo, CurSumInfo}), + SubPid = element(2, lists:nth(1, element(2, CurMemInfo))) - element(2, lists:nth(1, element(2, OldMemInfo))), + SubSum = element(2, lists:keyfind(total, 1, CurSumInfo)) - element(2, lists:keyfind(total, 1, OldSumInfo)), + case erlang:abs(SubSum) >= Threshold orelse erlang:abs(SubPid) >= Threshold of true -> - io:format("IMY*********Memory use changes are too large~n~p~n~p~n~p~n~p~n", [{addOrSub, Sub}, {timeDiff, CurTime - OldTime}, {old, OldModule, OldLine, OldTime, OldMemInfo}, {cur, CurModule, CurLine, CurTime, CurMemInfo}]); + io:format( + "IMY*********Memory use changes are too large:~n" + "addOrSubSum:~20w~n" + "addOrSubPid:~20w~n" + "usedTimeDiff:~19w~n" + "oldLine:~w~n" + "CurLine:~w~n" + "&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&\n" + "OldSumInfo:~w~n" + "CurSumInfo:~w~n" + "&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&\n" + "OldPidInfo:~p~n" + "************************************************************************************~n" + "CurPidInfo:~p~n", [SubSum, SubPid, CurTime - OldTime, {old, OldModule, OldLine, OldTime}, {cur, CurModule, CurLine, CurTime}, OldSumInfo, CurSumInfo, OldMemInfo, CurMemInfo]); _ -> ignore end From 4b4c96d3cb787857afddbb96493878427a60f181 Mon Sep 17 00:00:00 2001 From: SisMaker <1713699517@qq.com> Date: Mon, 1 Mar 2021 12:48:54 +0800 Subject: [PATCH 2/4] =?UTF-8?q?ft:=E5=87=BD=E6=95=B0=E6=B7=BB=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/uuid/utUuid.erl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/uuid/utUuid.erl b/src/uuid/utUuid.erl index 742978c..1c5239d 100644 --- a/src/uuid/utUuid.erl +++ b/src/uuid/utUuid.erl @@ -4,6 +4,7 @@ randStr/1 %% 随机指定长度的字符串 , uuid/0 %% 获取唯一UUID , uuidHex/0 %% 获取hex格式的唯一UUID + , uuidHexBin/0 ]). -spec randStr(integer()) -> string(). @@ -28,3 +29,7 @@ uuid() -> -spec uuidHex() -> binary(). uuidHex() -> utMd5:getMd5Hex(term_to_binary({erlang:system_time(nanosecond), rand:uniform(134217727), make_ref()})). + +-spec uuidHexBin() -> binary(). +uuidHexBin() -> + utMd5:getMd5HexBin(term_to_binary({erlang:system_time(nanosecond), rand:uniform(134217727), make_ref()})). \ No newline at end of file From b8aa770b51c13067f08c7e7aa8e63c8b6e99f7a7 Mon Sep 17 00:00:00 2001 From: SisMaker <1713699517@qq.com> Date: Tue, 9 Mar 2021 11:26:27 +0800 Subject: [PATCH 3/4] =?UTF-8?q?rf:=20utTime=20import=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E6=95=B4=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/timeDate/utTime.erl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/timeDate/utTime.erl b/src/timeDate/utTime.erl index faa4c76..fc71408 100644 --- a/src/timeDate/utTime.erl +++ b/src/timeDate/utTime.erl @@ -1,7 +1,7 @@ -module(utTime). -include("utTime.hrl"). --import(calendar, [day_of_the_week/1, iso_week_number/1, date_to_gregorian_days/1]). +-import(calendar, [day_of_the_week/1, day_of_the_week/3, iso_week_number/1, date_to_gregorian_days/1]). -export([ now/0 %% 当前的时间戳 秒 @@ -137,27 +137,27 @@ curTime() -> %% 当前星期几 -spec weekDay() -> week(). weekDay() -> - calendar:day_of_the_week(erlang:date()). + day_of_the_week(erlang:date()). %% 计算Data是星期几 -spec weekDay(Date :: date()) -> week(). weekDay(Date) -> - calendar:day_of_the_week(Date). + day_of_the_week(Date). %% 计算 年 月 日 是星期几 -spec weekDay(Year :: year(), Month :: month(), Day :: day()) -> week(). weekDay(Year, Month, Day) -> - calendar:day_of_the_week(Year, Month, Day). + day_of_the_week(Year, Month, Day). %% 计算当前的星期周期 -spec weekCycle() -> yearWeekCycle(). weekCycle() -> - calendar:iso_week_number(erlang:date()). + iso_week_number(erlang:date()). %% 计算 Date 的星期周期 -spec weekCycle(Date :: date()) -> yearWeekCycle(). weekCycle(Date) -> - calendar:iso_week_number(Date). + iso_week_number(Date). %% 将秒单位的时间戳 转为本地 datetime() -spec secToLDateTime(Sec :: timestamp()) -> datetime(). @@ -712,7 +712,7 @@ timeToSecs({H, M, S}) -> %% 计算 Date为该年的哪一天 -spec daysInYear(date()) -> integer(). daysInYear({Y, _, _} = Date) -> - calendar:date_to_gregorian_days(Date) - calendar:date_to_gregorian_days({Y, 1, 1}). + date_to_gregorian_days(Date) - date_to_gregorian_days({Y, 1, 1}). %% Data to Str -spec dateToStr(date()) -> string(). From 5ff4556b8fdd6dd23360f25428329e909cf5738a Mon Sep 17 00:00:00 2001 From: SisMaker <1713699517@qq.com> Date: Wed, 17 Mar 2021 18:22:45 +0800 Subject: [PATCH 4/4] =?UTF-8?q?rf:=20=E4=BB=A3=E7=A0=81=E6=95=B4=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../utBeamToSrc.erl | 0 .../utKvsToBeam.erl | 0 src/{dynamicCompile => compile}/utMMake.erl | 0 .../utStrToBeam.erl | 0 src/measure/p1_prof.erl | 359 ++++++++++++++++++ 5 files changed, 359 insertions(+) rename src/{dynamicCompile => compile}/utBeamToSrc.erl (100%) rename src/{dynamicCompile => compile}/utKvsToBeam.erl (100%) rename src/{dynamicCompile => compile}/utMMake.erl (100%) rename src/{dynamicCompile => compile}/utStrToBeam.erl (100%) create mode 100644 src/measure/p1_prof.erl diff --git a/src/dynamicCompile/utBeamToSrc.erl b/src/compile/utBeamToSrc.erl similarity index 100% rename from src/dynamicCompile/utBeamToSrc.erl rename to src/compile/utBeamToSrc.erl diff --git a/src/dynamicCompile/utKvsToBeam.erl b/src/compile/utKvsToBeam.erl similarity index 100% rename from src/dynamicCompile/utKvsToBeam.erl rename to src/compile/utKvsToBeam.erl diff --git a/src/dynamicCompile/utMMake.erl b/src/compile/utMMake.erl similarity index 100% rename from src/dynamicCompile/utMMake.erl rename to src/compile/utMMake.erl diff --git a/src/dynamicCompile/utStrToBeam.erl b/src/compile/utStrToBeam.erl similarity index 100% rename from src/dynamicCompile/utStrToBeam.erl rename to src/compile/utStrToBeam.erl diff --git a/src/measure/p1_prof.erl b/src/measure/p1_prof.erl new file mode 100644 index 0000000..b265d73 --- /dev/null +++ b/src/measure/p1_prof.erl @@ -0,0 +1,359 @@ +%%%------------------------------------------------------------------- +%%% File : p1_prof.erl +%%% Author : Evgeniy Khramtsov +%%% Description : Handy wrapper around eprof and fprof +%%% +%%% Created : 23 Jan 2010 by Evgeniy Khramtsov +%%% +%%% +%%% ejabberd, Copyright (C) 2002-2021 ProcessOne +%%% +%%% This program is free software; you can redistribute it and/or +%%% modify it under the terms of the GNU General Public License as +%%% published by the Free Software Foundation; either version 2 of the +%%% License, or (at your option) any later version. +%%% +%%% This program is distributed in the hope that it will be useful, +%%% but WITHOUT ANY WARRANTY; without even the implied warranty of +%%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +%%% General Public License for more details. +%%% +%%% You should have received a copy of the GNU General Public License along +%%% with this program; if not, write to the Free Software Foundation, Inc., +%%% 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +%%% +%%%------------------------------------------------------------------- +-module(p1_prof). + +%% API +-export([eprof_start/0, eprof_stop/0, + eprof_start/1, fprof_apply/3, + fprof_start/0, fprof_start/1, + fprof_stop/0, fprof_analyze/0, + queue/0, queue/1, memory/0, memory/1, + reds/0, reds/1, trace/1, help/0, + q/0, m/0, r/0, q/1, m/1, r/1, + locks/0, locks/1]). + +-define(TRACE_FILE, "/tmp/fprof.trace"). +-define(ANALYSIS_FILE, "/tmp/fprof.analysis"). + +%%==================================================================== +%% API +%%==================================================================== +eprof_start() -> + eprof_start(get_procs()). + +eprof_start(Duration) when is_integer(Duration) -> + eprof_start(get_procs()), + timer:sleep(timer:seconds(Duration)), + eprof_stop(); +eprof_start([]) -> + {error, no_procs_found}; +eprof_start(Procs) -> + eprof:start(), + eprof:start_profiling(Procs). + +fprof_apply(M, F, A) -> + fprof:apply(M, F, A, [{file, ?TRACE_FILE}]), + fprof_analyze(). + +fprof_start() -> + fprof_start(0). + +fprof_start(Duration) -> + case get_procs() of + [] -> + {error, no_procs_found}; + Procs -> + case fprof:trace([start, {procs, Procs}, {file, ?TRACE_FILE}]) of + ok -> + io:format("Profiling started, writing trace data to ~s~n", + [?TRACE_FILE]), + if Duration > 0 -> + timer:sleep(Duration*1000), + fprof:trace([stop]), + fprof:stop(); + true-> + ok + end; + Err -> + io:format("Couldn't start profiling: ~p~n", [Err]), + Err + end + end. + +fprof_stop() -> + fprof:trace([stop]), + case fprof:profile([{file, ?TRACE_FILE}]) of + ok -> + case fprof:analyse([totals, no_details, {sort, own}, + no_callers, {dest, ?ANALYSIS_FILE}]) of + ok -> + fprof:stop(), + format_fprof_analyze(); + Err -> + io:format("Couldn't analyze: ~p~n", [Err]), + Err + end; + Err -> + io:format("Couldn't compile a trace into profile data: ~p~n", + [Err]), + Err + end. + +fprof_analyze() -> + fprof_stop(). + +eprof_stop() -> + eprof:stop_profiling(), + eprof:analyze(total). + +help() -> + M = ?MODULE, + io:format("Brief help:~n" + "~p:queue(N) - show top N pids sorted by queue length~n" + "~p:queue() - shorthand for ~p:queue(10)~n" + "~p:memory(N) - show top N pids sorted by memory usage~n" + "~p:memory() - shorthand for ~p:memory(10)~n" + "~p:reds(N) - show top N pids sorted by reductions~n" + "~p:reds() - shorthand for ~p:reds(10)~n" + "~p:q(N)|~p:q() - same as ~p:queue(N)|~p:queue()~n" + "~p:m(N)|~p:m() - same as ~p:memory(N)|~p:memory()~n" + "~p:r(N)|~p:r() - same as ~p:reds(N)|~p:reds()~n" + "~p:trace(Pid) - trace Pid; to stop tracing close " + "Erlang shell with Ctrl+C~n" + "~p:eprof_start() - start eprof on all available pids; " + "DO NOT use on production system!~n" + "~p:eprof_stop() - stop eprof and print result~n" + "~p:fprof_start() - start fprof on all available pids; " + "DO NOT use on production system!~n" + "~p:fprof_stop() - stop eprof and print formatted result~n" + "~p:fprof_start(N) - start and run fprof for N seconds; " + "use ~p:fprof_analyze() to analyze collected statistics and " + "print formatted result; use on production system with CARE~n" + "~p:fprof_analyze() - analyze previously collected statistics " + "using ~p:fprof_start(N) and print formatted result~n" + "~p:help() - print this help~n", + lists:duplicate(31, M)). + +q() -> + queue(). + +q(N) -> + queue(N). + +m() -> + memory(). + +m(N) -> + memory(N). + +r() -> + reds(). + +r(N) -> + reds(N). + +queue() -> + queue(10). + +memory() -> + memory(10). + +reds() -> + reds(10). + +queue(N) -> + dump(N, lists:reverse(lists:ukeysort(1, all_pids(queue)))). + +memory(N) -> + dump(N, lists:reverse(lists:ukeysort(2, all_pids(memory)))). + +reds(N) -> + dump(N, lists:reverse(lists:ukeysort(3, all_pids(reductions)))). + +trace(Pid) -> + erlang:trace(Pid, true, [send, 'receive']), + trace_loop(). + +trace_loop() -> + receive + M -> + io:format("~p~n", [M]), + trace_loop() + end. + +%%==================================================================== +%% Internal functions +%%==================================================================== +get_procs() -> + processes(). + +format_fprof_analyze() -> + case file:consult(?ANALYSIS_FILE) of + {ok, [_, [{totals, _, _, TotalOWN}] | Rest]} -> + OWNs = lists:flatmap( + fun({MFA, _, _, OWN}) -> + Percent = OWN*100/TotalOWN, + case round(Percent) of + 0 -> + []; + _ -> + [{mfa_to_list(MFA), Percent}] + end + end, Rest), + ACCs = collect_accs(Rest), + MaxACC = find_max(ACCs), + MaxOWN = find_max(OWNs), + io:format("=== Sorted by OWN:~n"), + lists:foreach( + fun({MFA, Per}) -> + L = length(MFA), + S = lists:duplicate(MaxOWN - L + 2, $ ), + io:format("~s~s~.2f%~n", [MFA, S, Per]) + end, lists:reverse(lists:keysort(2, OWNs))), + io:format("~n=== Sorted by ACC:~n"), + lists:foreach( + fun({MFA, Per}) -> + L = length(MFA), + S = lists:duplicate(MaxACC - L + 2, $ ), + io:format("~s~s~.2f%~n", [MFA, S, Per]) + end, lists:reverse(lists:keysort(2, ACCs))); + Err -> + Err + end. + +mfa_to_list({M, F, A}) -> + atom_to_list(M) ++ ":" ++ atom_to_list(F) ++ "/" ++ integer_to_list(A); +mfa_to_list(F) when is_atom(F) -> + atom_to_list(F). + +find_max(List) -> + find_max(List, 0). + +find_max([{V, _}|Tail], Acc) -> + find_max(Tail, lists:max([length(V), Acc])); +find_max([], Acc) -> + Acc. + +collect_accs(List) -> + List1 = lists:filter( + fun({MFA, _, _, _}) -> + case MFA of + {sys, _, _} -> + false; + suspend -> + false; + {gen_fsm, _, _} -> + false; + {p1_fsm, _, _} -> + false; + {gen, _, _} -> + false; + {gen_server, _, _} -> + false; + {proc_lib, _, _} -> + false; + _ -> + true + end + end, List), + TotalACC = lists:sum([A || {_, _, A, _} <- List1]), + lists:flatmap( + fun({MFA, _, ACC, _}) -> + Percent = ACC*100/TotalACC, + case round(Percent) of + 0 -> + []; + _ -> + [{mfa_to_list(MFA), Percent}] + end + end, List1). + +all_pids(Type) -> + lists:foldl( + fun(P, Acc) when P == self() -> + %% exclude ourself from statistics + Acc; + (P, Acc) -> + case catch process_info( + P, + [message_queue_len, + status, + memory, + reductions, + dictionary, + current_function, + registered_name]) of + [{_, QLen}, {_, Status}, {_, Memory}, {_, Reds}, + {_, Dict}, {_, CurFun}, {_, RegName}] -> + Dict1 = filter_dict(Dict, RegName), + {IntQLen, Dict2} = + case lists:keytake('$internal_queue_len', 1, Dict1) of + {value, {_, N}, D} -> + {N, D}; + false -> + {0, Dict1} + end, + Len = QLen + IntQLen, + if Type == queue andalso Len == 0 -> + Acc; + true -> + Dict3 = [{message_queue_len, Len}, + {status, Status}, + {memory, Memory}, + {reductions, Reds}, + {current_function, CurFun}, + {registered_name, RegName}|Dict2], + [{Len, Memory, Reds, P, Dict3}|Acc] + end; + _ -> + Acc + end + end, [], processes()). + +dump(N, Rs) -> + lists:foreach( + fun({_, _, _, Pid, Properties}) -> + PidStr = pid_to_list(Pid), + [_, Maj, Min] = string:tokens( + string:substr( + PidStr, 2, length(PidStr) - 2), "."), + io:put_chars( + [io_lib:format("** pid: pid(0,~s,~s)~n", [Maj, Min]), + [io_lib:format("** ~s: ~p~n", [Key, Val]) + || {Key, Val} <- Properties], io_lib:nl()]) + end, nthhead(N, Rs)). + +nthhead(N, L) -> + lists:reverse(nthhead(N, L, [])). + +nthhead(0, _L, Acc) -> + Acc; +nthhead(N, [H|T], Acc) -> + nthhead(N-1, T, [H|Acc]); +nthhead(_N, [], Acc) -> + Acc. + +filter_dict(Dict, RegName) -> + lists:filter( + fun({'$internal_queue_len', _}) -> true; + ({'$initial_call', _}) -> RegName == []; + ({'$ancestors', _}) -> RegName == []; + (_) -> false + end, Dict). + +% output in the console counts of locks, optionally waiting for few seconds before collect +locks() -> + locks(5). +locks(Time) -> + lcnt:rt_opt({copy_save, true}), + lcnt:start(), + lcnt:clear(), + timer:sleep(Time*1000), + lcnt:collect(), + lcnt:conflicts(), + lcnt:stop(), + lcnt:rt_opt({copy_save, false}), + ok.