Merge remote-tracking branch 'origin/master' into master

src/measure/p1_prof.erl

@@ -0,0 +1,359 @@
+%%%-------------------------------------------------------------------
+%%% File    : p1_prof.erl
+%%% Author  : Evgeniy Khramtsov <ekhramtsov@process-one.net>
+%%% Description : Handy wrapper around eprof and fprof
+%%%
+%%% Created : 23 Jan 2010 by Evgeniy Khramtsov <ekhramtsov@process-one.net>
+%%%
+%%%
+%%% ejabberd, Copyright (C) 2002-2021   ProcessOne
+%%%
+%%% This program is free software; you can redistribute it and/or
+%%% modify it under the terms of the GNU General Public License as
+%%% published by the Free Software Foundation; either version 2 of the
+%%% License, or (at your option) any later version.
+%%%
+%%% This program is distributed in the hope that it will be useful,
+%%% but WITHOUT ANY WARRANTY; without even the implied warranty of
+%%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+%%% General Public License for more details.
+%%%
+%%% You should have received a copy of the GNU General Public License along
+%%% with this program; if not, write to the Free Software Foundation, Inc.,
+%%% 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+%%%
+%%%-------------------------------------------------------------------
+-module(p1_prof).
+
+%% API
+-export([eprof_start/0, eprof_stop/0,
+	 eprof_start/1, fprof_apply/3,
+	 fprof_start/0, fprof_start/1,
+	 fprof_stop/0, fprof_analyze/0,
+	 queue/0, queue/1, memory/0, memory/1,
+	 reds/0, reds/1, trace/1, help/0,
+	 q/0, m/0, r/0, q/1, m/1, r/1,
+	 locks/0, locks/1]).
+
+-define(TRACE_FILE, "/tmp/fprof.trace").
+-define(ANALYSIS_FILE, "/tmp/fprof.analysis").
+
+%%====================================================================
+%% API
+%%====================================================================
+eprof_start() ->
+    eprof_start(get_procs()).
+
+eprof_start(Duration) when is_integer(Duration) ->
+    eprof_start(get_procs()),
+    timer:sleep(timer:seconds(Duration)),
+    eprof_stop();
+eprof_start([]) ->
+    {error, no_procs_found};
+eprof_start(Procs) ->
+    eprof:start(),
+    eprof:start_profiling(Procs).
+
+fprof_apply(M, F, A) ->
+    fprof:apply(M, F, A, [{file, ?TRACE_FILE}]),
+    fprof_analyze().
+
+fprof_start() ->
+    fprof_start(0).
+
+fprof_start(Duration) ->
+    case get_procs() of
+        [] ->
+            {error, no_procs_found};
+        Procs ->
+            case fprof:trace([start, {procs, Procs}, {file, ?TRACE_FILE}]) of
+                ok ->
+                    io:format("Profiling started, writing trace data to ~s~n",
+                              [?TRACE_FILE]),
+                    if Duration > 0 ->
+                            timer:sleep(Duration*1000),
+                            fprof:trace([stop]),
+                            fprof:stop();
+                       true->
+                            ok
+                    end;
+                Err ->
+                    io:format("Couldn't start profiling: ~p~n", [Err]),
+                    Err
+            end
+    end.
+
+fprof_stop() ->
+    fprof:trace([stop]),
+    case fprof:profile([{file, ?TRACE_FILE}]) of
+        ok ->
+            case fprof:analyse([totals, no_details, {sort, own},
+                                no_callers, {dest, ?ANALYSIS_FILE}]) of
+                ok ->
+                    fprof:stop(),
+                    format_fprof_analyze();
+                Err ->
+                    io:format("Couldn't analyze: ~p~n", [Err]),
+                    Err
+            end;
+        Err ->
+            io:format("Couldn't compile a trace into profile data: ~p~n",
+                      [Err]),
+            Err
+    end.
+
+fprof_analyze() ->
+    fprof_stop().
+
+eprof_stop() ->
+    eprof:stop_profiling(),
+    eprof:analyze(total).
+
+help() ->
+    M = ?MODULE,
+    io:format("Brief help:~n"
+	      "~p:queue(N) - show top N pids sorted by queue length~n"
+	      "~p:queue() - shorthand for ~p:queue(10)~n"
+	      "~p:memory(N) - show top N pids sorted by memory usage~n"
+	      "~p:memory() - shorthand for ~p:memory(10)~n"
+	      "~p:reds(N) - show top N pids sorted by reductions~n"
+	      "~p:reds() - shorthand for ~p:reds(10)~n"
+	      "~p:q(N)|~p:q() - same as ~p:queue(N)|~p:queue()~n"
+	      "~p:m(N)|~p:m() - same as ~p:memory(N)|~p:memory()~n"
+	      "~p:r(N)|~p:r() - same as ~p:reds(N)|~p:reds()~n"
+	      "~p:trace(Pid) - trace Pid; to stop tracing close "
+	      "Erlang shell with Ctrl+C~n"
+	      "~p:eprof_start() - start eprof on all available pids; "
+	      "DO NOT use on production system!~n"
+	      "~p:eprof_stop() - stop eprof and print result~n"
+	      "~p:fprof_start() - start fprof on all available pids; "
+	      "DO NOT use on production system!~n"
+	      "~p:fprof_stop() - stop eprof and print formatted result~n"
+	      "~p:fprof_start(N) - start and run fprof for N seconds; "
+	      "use ~p:fprof_analyze() to analyze collected statistics and "
+	      "print formatted result; use on production system with CARE~n"
+	      "~p:fprof_analyze() - analyze previously collected statistics "
+	      "using ~p:fprof_start(N) and print formatted result~n"
+	      "~p:help() - print this help~n",
+	      lists:duplicate(31, M)).
+
+q() ->
+    queue().
+
+q(N) ->
+    queue(N).
+
+m() ->
+    memory().
+
+m(N) ->
+    memory(N).
+
+r() ->
+    reds().
+
+r(N) ->
+    reds(N).
+
+queue() ->
+    queue(10).
+
+memory() ->
+    memory(10).
+
+reds() ->
+    reds(10).
+
+queue(N) ->
+    dump(N, lists:reverse(lists:ukeysort(1, all_pids(queue)))).
+
+memory(N) ->
+    dump(N, lists:reverse(lists:ukeysort(2, all_pids(memory)))).
+
+reds(N) ->
+    dump(N, lists:reverse(lists:ukeysort(3, all_pids(reductions)))).
+
+trace(Pid) ->
+    erlang:trace(Pid, true, [send, 'receive']),
+    trace_loop().
+
+trace_loop() ->
+    receive
+	M ->
+	    io:format("~p~n", [M]),
+	    trace_loop()
+    end.
+
+%%====================================================================
+%% Internal functions
+%%====================================================================
+get_procs() ->
+    processes().
+
+format_fprof_analyze() ->
+    case file:consult(?ANALYSIS_FILE) of
+	{ok, [_, [{totals, _, _, TotalOWN}] | Rest]} ->
+	    OWNs = lists:flatmap(
+		     fun({MFA, _, _, OWN}) ->
+			     Percent = OWN*100/TotalOWN,
+			     case round(Percent) of
+				 0 ->
+				     [];
+				 _ ->
+				     [{mfa_to_list(MFA), Percent}]
+			     end
+		     end, Rest),
+	    ACCs = collect_accs(Rest),
+	    MaxACC = find_max(ACCs),
+	    MaxOWN = find_max(OWNs),
+	    io:format("=== Sorted by OWN:~n"),
+	    lists:foreach(
+	      fun({MFA, Per}) ->
+		      L = length(MFA),
+		      S = lists:duplicate(MaxOWN - L + 2, $ ),
+		      io:format("~s~s~.2f%~n", [MFA, S, Per])
+	      end, lists:reverse(lists:keysort(2, OWNs))),
+	    io:format("~n=== Sorted by ACC:~n"),
+	    lists:foreach(
+	      fun({MFA, Per}) ->
+		      L = length(MFA),
+		      S = lists:duplicate(MaxACC - L + 2, $ ),
+		      io:format("~s~s~.2f%~n", [MFA, S, Per])
+	      end, lists:reverse(lists:keysort(2, ACCs)));
+	Err ->
+	    Err
+    end.
+
+mfa_to_list({M, F, A}) ->
+    atom_to_list(M) ++ ":" ++ atom_to_list(F) ++ "/" ++ integer_to_list(A);
+mfa_to_list(F) when is_atom(F) ->
+    atom_to_list(F).
+
+find_max(List) ->
+    find_max(List, 0).
+
+find_max([{V, _}|Tail], Acc) ->
+    find_max(Tail, lists:max([length(V), Acc]));
+find_max([], Acc) ->
+    Acc.
+
+collect_accs(List) ->
+    List1 = lists:filter(
+	      fun({MFA, _, _, _}) ->
+		      case MFA of
+			  {sys, _, _} ->
+			      false;
+			  suspend ->
+			      false;
+			  {gen_fsm, _, _} ->
+			      false;
+			  {p1_fsm, _, _} ->
+			      false;
+			  {gen, _, _} ->
+			      false;
+			  {gen_server, _, _} ->
+			      false;
+			  {proc_lib, _, _} ->
+			      false;
+			  _ ->
+			      true
+		      end
+	      end, List),
+    TotalACC = lists:sum([A || {_, _, A, _} <- List1]),
+    lists:flatmap(
+      fun({MFA, _, ACC, _}) ->
+	      Percent = ACC*100/TotalACC,
+	      case round(Percent) of
+		  0 ->
+		      [];
+		  _ ->
+		      [{mfa_to_list(MFA), Percent}]
+	      end
+      end, List1).
+
+all_pids(Type) ->
+    lists:foldl(
+      fun(P, Acc) when P == self() ->
+	      %% exclude ourself from statistics
+	      Acc;
+	 (P, Acc) ->
+	      case catch process_info(
+			   P,
+			   [message_queue_len,
+			    status,
+			    memory,
+			    reductions,
+			    dictionary,
+			    current_function,
+			    registered_name]) of
+		  [{_, QLen}, {_, Status}, {_, Memory}, {_, Reds},
+		   {_, Dict}, {_, CurFun}, {_, RegName}] ->
+		      Dict1 = filter_dict(Dict, RegName),
+                      {IntQLen, Dict2} =
+			  case lists:keytake('$internal_queue_len', 1, Dict1) of
+			      {value, {_, N}, D} ->
+				  {N, D};
+			      false ->
+				  {0, Dict1}
+			  end,
+		      Len = QLen + IntQLen,
+		      if Type == queue andalso Len == 0 ->
+			      Acc;
+			 true ->
+                              Dict3 = [{message_queue_len, Len},
+				       {status, Status},
+				       {memory, Memory},
+				       {reductions, Reds},
+				       {current_function, CurFun},
+				       {registered_name, RegName}|Dict2],
+			      [{Len, Memory, Reds, P, Dict3}|Acc]
+		      end;
+		  _ ->
+		      Acc
+	      end
+      end, [], processes()).
+
+dump(N, Rs) ->
+    lists:foreach(
+      fun({_, _, _, Pid, Properties}) ->
+	      PidStr = pid_to_list(Pid),
+	      [_, Maj, Min] = string:tokens(
+				string:substr(
+				  PidStr, 2, length(PidStr) - 2), "."),
+	      io:put_chars(
+		[io_lib:format("** pid: pid(0,~s,~s)~n", [Maj, Min]),
+		 [io_lib:format("** ~s: ~p~n", [Key, Val])
+		  || {Key, Val} <- Properties], io_lib:nl()])
+      end, nthhead(N, Rs)).
+
+nthhead(N, L) ->
+    lists:reverse(nthhead(N, L, [])).
+
+nthhead(0, _L, Acc) ->
+    Acc;
+nthhead(N, [H|T], Acc) ->
+    nthhead(N-1, T, [H|Acc]);
+nthhead(_N, [], Acc) ->
+    Acc.
+
+filter_dict(Dict, RegName) ->
+    lists:filter(
+      fun({'$internal_queue_len', _}) -> true;
+	 ({'$initial_call', _}) -> RegName == [];
+	 ({'$ancestors', _}) -> RegName == [];
+	 (_) -> false
+      end, Dict).
+
+% output in the console counts of locks, optionally waiting for few seconds before collect
+locks() ->
+    locks(5).
+locks(Time) ->
+    lcnt:rt_opt({copy_save, true}),
+    lcnt:start(),
+    lcnt:clear(),
+    timer:sleep(Time*1000),
+    lcnt:collect(),
+    lcnt:conflicts(),
+    lcnt:stop(),
+    lcnt:rt_opt({copy_save, false}),
+    ok.

src/srvNodeMgr/utVMInfo.erl

@@ -464,22 +464,35 @@ get_processes() ->
 		get_process_info_and_large_than_value(memory, 0)).
 
 memInfoInit(CurModule, CurLine) ->
-	erlang:put(?pdMemInfo, {CurModule, CurLine, erlang:system_time(nanosecond), recon:info(self(), memory_used)}).
+	erlang:put(?pdMemInfo, {CurModule, CurLine, erlang:system_time(nanosecond), recon:info(self(), memory_used), erlang:memory()}).
 
 memInfoPrint(CurModule, CurLine, Threshold) ->
 	case erlang:get(?pdMemInfo) of
 		undefined ->
-			erlang:put(?pdMemInfo, {CurModule, CurLine, erlang:system_time(nanosecond), recon:info(self(), memory_used)});
-		{OldModule, OldLine, OldTime, OldMemInfo} ->
+			erlang:put(?pdMemInfo, {CurModule, CurLine, erlang:system_time(nanosecond), recon:info(self(), memory_used), erlang:memory()});
+		{OldModule, OldLine, OldTime, OldMemInfo, OldSumInfo} ->
 			CurMemInfo = recon:info(self(), memory_used),
 			CurTime = erlang:system_time(nanosecond),
-			erlang:put(?pdMemInfo, {CurModule, CurLine, CurTime, recon:info(self(), memory_used)}),
-			OldUsed = element(2, lists:nth(1, element(2, OldMemInfo))),
-			CurUsed = element(2, lists:nth(1, element(2, CurMemInfo))),
-			Sub = CurUsed - OldUsed,
-			case erlang:abs(Sub) >= Threshold of
+			CurSumInfo = erlang:memory(),
+			erlang:put(?pdMemInfo, {CurModule, CurLine, CurTime, CurMemInfo, CurSumInfo}),
+			SubPid = element(2, lists:nth(1, element(2, CurMemInfo))) - element(2, lists:nth(1, element(2, OldMemInfo))),
+			SubSum = element(2, lists:keyfind(total, 1, CurSumInfo)) - element(2, lists:keyfind(total, 1, OldSumInfo)),
+			case erlang:abs(SubSum) >= Threshold orelse erlang:abs(SubPid) >= Threshold of
 				true ->
-					io:format("IMY*********Memory use changes are too large~n~p~n~p~n~p~n~p~n", [{addOrSub, Sub}, {timeDiff, CurTime - OldTime}, {old, OldModule, OldLine, OldTime, OldMemInfo}, {cur, CurModule, CurLine, CurTime, CurMemInfo}]);
+					io:format(
+						"IMY*********Memory use changes are too large:~n"
+						"addOrSubSum:~20w~n"
+						"addOrSubPid:~20w~n"
+						"usedTimeDiff:~19w~n"
+						"oldLine:~w~n"
+						"CurLine:~w~n"
+						"&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&\n"
+						"OldSumInfo:~w~n"
+						"CurSumInfo:~w~n"
+						"&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&\n"
+						"OldPidInfo:~p~n"
+						"************************************************************************************~n"
+						"CurPidInfo:~p~n", [SubSum, SubPid, CurTime - OldTime, {old, OldModule, OldLine, OldTime}, {cur, CurModule, CurLine, CurTime}, OldSumInfo, CurSumInfo, OldMemInfo, CurMemInfo]);
 				_ ->
 					ignore
 			end

src/timeDate/utTime.erl

@@ -1,7 +1,7 @@
 -module(utTime).
 -include("utTime.hrl").
 
--import(calendar, [day_of_the_week/1, iso_week_number/1, date_to_gregorian_days/1]).
+-import(calendar, [day_of_the_week/1, day_of_the_week/3, iso_week_number/1, date_to_gregorian_days/1]).
 
 -export([
    now/0                            %% 当前的时间戳 秒
@@ -137,27 +137,27 @@ curTime() ->
 %% 当前星期几
 -spec weekDay() -> week().
 weekDay() ->
-   calendar:day_of_the_week(erlang:date()).
+   day_of_the_week(erlang:date()).
 
 %% 计算Data是星期几
 -spec weekDay(Date :: date()) -> week().
 weekDay(Date) ->
-   calendar:day_of_the_week(Date).
+   day_of_the_week(Date).
 
 %% 计算 年 月 日 是星期几
 -spec weekDay(Year :: year(), Month :: month(), Day :: day()) -> week().
 weekDay(Year, Month, Day) ->
-   calendar:day_of_the_week(Year, Month, Day).
+   day_of_the_week(Year, Month, Day).
 
 %% 计算当前的星期周期
 -spec weekCycle() -> yearWeekCycle().
 weekCycle() ->
-   calendar:iso_week_number(erlang:date()).
+   iso_week_number(erlang:date()).
 
 %% 计算 Date 的星期周期
 -spec weekCycle(Date :: date()) -> yearWeekCycle().
 weekCycle(Date) ->
-   calendar:iso_week_number(Date).
+   iso_week_number(Date).
 
 %% 将秒单位的时间戳 转为本地 datetime()
 -spec secToLDateTime(Sec :: timestamp()) -> datetime().
@@ -712,7 +712,7 @@ timeToSecs({H, M, S}) ->
 %% 计算 Date为该年的哪一天
 -spec daysInYear(date()) -> integer().
 daysInYear({Y, _, _} = Date) ->
-   calendar:date_to_gregorian_days(Date) - calendar:date_to_gregorian_days({Y, 1, 1}).
+   date_to_gregorian_days(Date) - date_to_gregorian_days({Y, 1, 1}).
 
 %%  Data to Str
 -spec dateToStr(date()) -> string().

src/uuid/utUuid.erl

@@ -4,6 +4,7 @@
    randStr/1            %% 随机指定长度的字符串
    , uuid/0               %% 获取唯一UUID
    , uuidHex/0            %% 获取hex格式的唯一UUID
+   , uuidHexBin/0
 ]).
 
 -spec randStr(integer()) -> string().
@@ -28,3 +29,7 @@ uuid() ->
 -spec uuidHex() -> binary().
 uuidHex() ->
    utMd5:getMd5Hex(term_to_binary({erlang:system_time(nanosecond), rand:uniform(134217727), make_ref()})).
+
+-spec uuidHexBin() -> binary().
+uuidHexBin() ->
+   utMd5:getMd5HexBin(term_to_binary({erlang:system_time(nanosecond), rand:uniform(134217727), make_ref()})).