ft: 优化

src/eMake.erl

@@ -1,46 +1,74 @@
 -module(eMake).
-%% 多进程编译,修改自otp/lib/tools/src/make.erl
-%% 解析Emakefile,根据获取{mods, options}列表,
-%% 按照次序编译每项(解决编译顺序的问题)
-%% 其中mods也可以包含多个模块,当大于1个时,
-%% 可以启动多个process进行编译,从而提高编译速度.
-
 -include_lib("kernel/include/file.hrl").
 
 -export([
    main/1
-   , all/3
+]).
+
+-export([
+   compileWorker/5
+   , readEMake/0
+   , saveEMake/1
 ]).
 
 -define(MakeOpts, [noexec, load, netload, noload]).
 
 main(Args) ->
-   io:format("~p~n", [Args]),
+   process_flag(trap_exit, true),
 
    case Args of
       [] ->
          all(max(1, erlang:system_info(schedulers) - 1), "./Emakefile", []);
-      [EMakeFileOrCnt] ->
-         try list_to_integer(EMakeFileOrCnt) of
+      [EMakeFileOrWorkCnt] ->
+         try list_to_integer(EMakeFileOrWorkCnt) of
             Cnt ->
                all(max(1, Cnt), "./Emakefile", [])
          catch _:_ ->
-            all(max(1, erlang:system_info(schedulers) - 1), EMakeFileOrCnt, [])
+            all(max(1, erlang:system_info(schedulers) - 1), EMakeFileOrWorkCnt, [])
          end;
-      [EMakeFile, CntStr] ->
-         all(max(1, list_to_integer(CntStr)), EMakeFile, []);
-      [EMakeFile, CntStr, OptsStr] ->
+      [EMakeFile, WorkCntStr] ->
+         all(max(1, list_to_integer(WorkCntStr)), EMakeFile, []);
+      [EMakeFile, WorkCntStr, OptsStr] ->
          {ok, Opts} = strToTerm(OptsStr),
-         all(max(1, list_to_integer(CntStr)), EMakeFile, Opts)
+         all(max(1, list_to_integer(WorkCntStr)), EMakeFile, Opts)
+   end.
+
+eMakeFile() ->
+   filename:join(code:root_dir(), ".eMake.log").
+
+readEMake() ->
+   try {ok, [LastTime]} = file:consult(eMakeFile()), LastTime of
+      Value ->
+         Value
+   catch _:_ ->
+      0
    end.
 
-all(Worker, EMakeFile, Opts) ->
+saveEMake(NowTime) ->
+   try file:write_file(eMakeFile(), <<(integer_to_binary(NowTime))/binary, ".">>)
+   catch _:_ ->
+      ok
+   end.
+
+all(WorkerCnt, EMakeFile, Opts) ->
+   StartTime = erlang:system_time(second),
    {MakeOpts, CompileOpts} = splitOpts(Opts, [], []),
    case readEMakefile(EMakeFile, CompileOpts) of
-      Files when is_list(Files) ->
-         do_make_files(Worker, Files, MakeOpts);
-      error ->
-         error
+      {ok, Files} ->
+         forMake(Files, WorkerCnt, lists:member(noexec, MakeOpts), load_opt(MakeOpts), []),
+         EndTime = erlang:system_time(second),
+         % LastTime = readEMake(),
+         % case LastTime /= 0 andalso StartTime < LastTime of
+         %    true ->
+         %       %% StartTime < LastTime 当前系统时间比历史编译时间小的时候 可能往回调过时间 所以要全编译
+         %       put(compile_all, 1);
+         %    _ ->
+         %       ignore
+         % end,
+         % saveEMake(EndTime),
+         io:format("compile over all is ok use time:~ps", [EndTime - StartTime]);
+      _Err ->
+         _Err
    end.
 
 splitOpts([], Make, Compile) ->
@@ -53,7 +81,6 @@ splitOpts([H | T], Make, Compile) ->
          splitOpts(T, Make, [H | Compile])
    end.
 
-
 %% term反序列化, string转换为term
 strToTerm(String) ->
    case erl_scan:string(String ++ ".") of
@@ -63,9 +90,22 @@ strToTerm(String) ->
          {error, _Err}
    end.
 
-do_make_files(Worker, Fs, Opts) ->
-   %io:format("worker:~p~nfs:~p~nopts:~p~n", [Worker, Fs, Opts]),
-   process(Fs, Worker, lists:member(noexec, Opts), load_opt(Opts)).
+%% Any flags that are not recognixed as make flags are passed directly
+%% to the compiler.
+%% So for example make:all([load,debug_info]) will make everything
+%% with the debug_info flag and load it.
+load_opt(Opts) ->
+   case lists:member(netload, Opts) of
+      true ->
+         netload;
+      false ->
+         case lists:member(load, Opts) of
+            true ->
+               load;
+            _ ->
+               noload
+         end
+   end.
 
 %%% 读取给定的 Emakefile 并返回一个元组列表： [{Mods,Opts}]
 %%% %%% Mods 是模块名称（字符串）的列表
@@ -73,16 +113,16 @@ do_make_files(Worker, Fs, Opts) ->
 readEMakefile(EMakefile, Opts) ->
    case file:consult(EMakefile) of
       {ok, EMake} ->
-         Ret  = transform(EMake, Opts, []),
+         Ret = transform(EMake, Opts, []),
          erase(),
-         Ret;
+         {ok, Ret};
       {error, enoent} ->
          %% 没有EMakefile 仅仅编译当前没有了下的文件 如果想要编译所有子目录下的文件 使用 filelib:wildcard("./**/*.erl")
          Mods = [filename:rootname(F) || F <- filelib:wildcard("*.erl")],
-         [{Mods, Opts}];
-      {error, Other} ->
+         {ok, [{Mods, Opts}]};
+      {error, Other} = _Err ->
          io:format("the Emakefile:~s is error:~p~n", [EMakefile, Other]),
-         error
+         _Err
    end.
 
 transform([], _Opts, Files) ->
@@ -92,7 +132,7 @@ transform([{Mod, ModOpts} | EMake], Opts, Files) ->
       [] ->
          transform(EMake, Opts, Files);
       Mods ->
-         transform(EMake, Opts, [{Mods, ModOpts ++ Opts} | Files])
+         transform(EMake, Opts, [{Mods, Opts ++ ModOpts} | Files])
    end;
 transform([Mod | EMake], Opts, Files) ->
    case expand(Mod) of
@@ -109,141 +149,151 @@ expand(Mods) when is_list(Mods), not is_integer(hd(Mods)) ->
 expand(Mod) ->
    case lists:member($*, Mod) of
       true ->
-         Fun = fun(F, Acc) ->
-            M = filename:rootname(F),
-            case get(M) of
-               undefined ->
-                  put(M, 1),
-                  [M | Acc];
-               _ ->
-                  Acc
-            end
-               end,
-         lists:foldl(Fun, [], filelib:wildcard(Mod ++ ".erl"));
+         foldErl(filelib:wildcard(Mod ++ ".erl"), []);
       _ ->
-         Mod2 = filename:rootname(Mod, ".erl"),
-         case get(Mod2) of
+         M = filename:rootname(Mod, ".erl"),
+         case get(M) of
             undefined ->
                put(M, 1),
-               [Mod2];
+               [M];
             _ ->
                []
          end
    end.
 
-%% Any flags that are not recognixed as make flags are passed directly
-%% to the compiler.
-%% So for example make:all([load,debug_info]) will make everything
-%% with the debug_info flag and load it.
-load_opt(Opts) ->
-   case lists:member(netload, Opts) of
-      true ->
-         netload;
-      false ->
-         case lists:member(load, Opts) of
+foldErl([], Acc) ->
+   Acc;
+foldErl([OneFile | Left], Acc) ->
+   M = filename:rootname(OneFile),
+   case get(M) of
+      undefined ->
+         put(M, 1),
+         foldErl(Left, [M | Acc]);
+      _ ->
+         foldErl(Left, Acc)
+   end.
+
+-define(OnceCnt, 8).
+forMake([], _Worker, _NoExec, _Load, AllWorkPids) ->
+   receive
+      {mOverCompile, WPid} ->
+         NewAllWorkPids = lists:delete(WPid, AllWorkPids),
+         case NewAllWorkPids of
+            [] ->
+               ok;
+            _ ->
+               forMake([], _Worker, _NoExec, _Load, NewAllWorkPids)
+         end;
+      {mCompileError, Err} ->
+         errorStop(Err, AllWorkPids);
+      _Other ->
+         io:format("forMake [] receive unexpect msg:~p ~n", [_Other])
+   end;
+forMake([{Mods, Opts} | Rest], Worker, NoExec, Load, AllWorkPids) ->
+   case Mods of
+      [] ->
+         forMake(Rest, Worker, NoExec, Load, AllWorkPids);
+      _ ->
+         case Worker > 0 of
             true ->
-               load;
+               {Files, More} = splitMods(Mods),
+               WPid = spawn_link(?MODULE, compileWorker, [Files, Opts, self(), NoExec, Load]),
+               case More of
+                  over ->
+                     forMake(Rest, Worker - 1, NoExec, Load, [WPid | AllWorkPids]);
+                  _ ->
+                     forMake([{More, Opts} | Rest], Worker - 1, NoExec, Load, [WPid | AllWorkPids])
+               end;
             _ ->
-               noload
+               receive
+                  {mOverCompile, WPid} ->
+                     {Files, More} = splitMods(Mods),
+                     erlang:send(WPid, {mNewFile, Files, Opts}),
+                     case More of
+                        over ->
+                           forMake(Rest, Worker, NoExec, Load, AllWorkPids);
+                        _ ->
+                           forMake([{More, Opts} | Rest], Worker, NoExec, Load, AllWorkPids)
+                     end;
+                  {mCompileError, Err} ->
+                     errorStop(Err, AllWorkPids);
+                  _Other ->
+                     io:format("forMake xx receive unexpect msg:~p ~n", [_Other])
+               end
          end
    end.
 
-%% 处理
-process([{[], _Opts} | Rest], Worker, NoExec, Load) ->
-   process(Rest, Worker, NoExec, Load);
-process([{L, Opts} | Rest], Worker, NoExec, Load) ->
-   Len = length(L),
-   Worker2 = erlang:min(Len, Worker),
-   case catch do_worker(L, Opts, NoExec, Load, Worker2) of
-      error ->
-         error;
-      ok ->
-         process(Rest, Worker, NoExec, Load)
-   end;
-process([], _Worker, _NoExec, _Load) ->
-   up_to_date.
-
-do_worker(L, Opts, NoExec, Load, Worker) ->
-
-   %% 先开启worker个编译进程
-   SplitNum = min(length(L), Worker),
-   {L1, L2} = lists:split(SplitNum, L),
+splitMods(Mods) ->
+   case length(Mods) =< ?OnceCnt of
+      true ->
+         {Mods, over};
+      _ ->
+         lists:split(?OnceCnt, Mods)
+   end.
 
-   % 启动进程
-   Ref = make_ref(),
-   Pids =
-      [begin
-          start_worker([E], Opts, NoExec, Load, self(), Ref)
-       end || E <- L1],
-   do_wait_worker(length(Pids), L2, Opts, NoExec, Load, Ref).
+errorStop(Err, AllWorkPids) ->
+   [exit(OnePid, kill) || OnePid <- AllWorkPids],
+   case Err of
+      {File, Errors, Warnings} ->
+         io:format("the file:~ts compile error:~p wrar:~p", [File, Errors, Warnings]);
+      File ->
+         io:format("the file:~ts compile error please check", [File])
+   end.
 
-%% @doc 一个文件编译完成后，立即进行下一个文件编译,不用等待
-do_wait_worker(0, [], _Opts, _NoExec, _Load, _Ref) ->
-   ok;
-do_wait_worker(N, L, Opts, NoExec, Load, Ref) ->
+compileWorker([], _Opts, Parent, NoExec, Load) ->
+   erlang:send(Parent, {mOverCompile, self()}),
    receive
-      {ack, Ref} ->
-         case L of
-            [H | T] ->
-               %% 未编译完成，编译后面的文件
-               start_worker(H, Opts, NoExec, Load, self(), Ref),
-               do_wait_worker(N, T, Opts, NoExec, Load, Ref);
-            [] ->
-               %% 等待所有结果返回
-               do_wait_worker(N - 1, [], Opts, NoExec, Load, Ref)
-         end;
-      {error, Ref} ->
-         throw(error);
-      {'EXIT', _P, _Reason} ->
-         do_wait_worker(N, L, Opts, NoExec, Load, Ref);
+      {mNewFile, Files, Opts} ->
+         compileWorker(Files, Opts, Parent, NoExec, Load);
       _Other ->
-         io:format("receive unknown msg:~p~n", [_Other]),
-         do_wait_worker(N, L, Opts, NoExec, Load, Ref)
+         io:format("compileWorker [] receive unexpect msg:~p ~n", [_Other])
+   end;
+compileWorker([OneFile | Files], Opts, Parent, NoExec, Load) ->
+   case compile(coerce_2_list(OneFile), NoExec, Load, Opts) of
+      error ->
+         Parent ! {mCompileError, OneFile},
+         exit(error);
+      {error, Errors, Warnings} ->
+         Parent ! {mCompileError, {OneFile, Errors, Warnings}},
+         exit(error);
+      _ ->
+         compileWorker(Files, Opts, Parent, NoExec, Load)
    end.
 
-start_worker(F, Opts, NoExec, Load, Parent, Ref) ->
-   Fun =
-      fun() ->
-         case recompilep(coerce_2_list(F), NoExec, Load, Opts) of
-            error ->
-               Parent ! {error, Ref},
-               exit(error);
-            _ ->
-               ok
-         end,
-         Parent ! {ack, Ref}
-      end,
-   spawn_link(Fun).
-
-recompilep(File, NoExec, Load, Opts) ->
-   ObjName = lists:append(filename:basename(File), code:objfile_extension()),
-   ObjFile = case lists:keysearch(outdir, 1, Opts) of
-                {value, {outdir, OutDir}} ->
-                   filename:join(coerce_2_list(OutDir), ObjName);
-                false ->
-                   ObjName
-             end,
-   case exists(ObjFile) of
-      true ->
-         recompilep1(File, NoExec, Load, Opts, ObjFile);
-      false ->
+compile(File, NoExec, Load, Opts) ->
+   case get(compile_all) of
+      undefined ->
+         ObjName = lists:append(filename:basename(File), code:objfile_extension()),
+         ObjFile =
+            case lists:keysearch(outdir, 1, Opts) of
+               {value, {outdir, OutDir}} ->
+                  filename:join(coerce_2_list(OutDir), ObjName);
+               false ->
+                  ObjName
+            end,
+         case exists(ObjFile) of
+            true ->
+               reCompile(File, NoExec, Load, Opts, ObjFile);
+            false ->
+               recompile(File, NoExec, Load, Opts)
+         end;
+      _ ->
          recompile(File, NoExec, Load, Opts)
    end.
 
-recompilep1(File, NoExec, Load, Opts, ObjFile) ->
-   {ok, Erl} = file:read_file_info(lists:append(File, ".erl")),
-   {ok, Obj} = file:read_file_info(ObjFile),
-   recompilep1(Erl, Obj, File, NoExec, Load, Opts).
-
-recompilep1(#file_info{mtime = Te},
-   #file_info{mtime = To}, File, NoExec, Load, Opts) when Te > To ->
-   recompile(File, NoExec, Load, Opts);
-recompilep1(_Erl, #file_info{mtime = To}, File, NoExec, Load, Opts) ->
-   recompile2(To, File, NoExec, Load, Opts).
+reCompile(File, NoExec, Load, Opts, ObjFile) ->
+   {ok, #file_info{mtime = SrcTime}} = file:read_file_info(lists:append(File, ".erl")),
+   {ok, #file_info{mtime = ObjTime}} = file:read_file_info(ObjFile),
+   case SrcTime > ObjTime of
+      true ->
+         recompile(File, NoExec, Load, Opts);
+      _ ->
+         ckIncludeRecompile(ObjTime, File, NoExec, Load, Opts)
+   end.
 
 %% recompile2(ObjMTime, File, NoExec, Load, Opts)
 %% Check if file is of a later date than include files.
-recompile2(ObjMTime, File, NoExec, Load, Opts) ->
+ckIncludeRecompile(ObjMTime, File, NoExec, Load, Opts) ->
    IncludePath = include_opt(Opts),
    case check_includes(lists:append(File, ".erl"), IncludePath, ObjMTime) of
       true ->
@@ -264,15 +314,15 @@ include_opt([]) ->
 %% Where load can be netload | load | noload
 
 recompile(File, true, _Load, _Opts) ->
-   io:format("Out of date: 
~s\n", [File]);
+   io:format("Out of date: ~ts\n", [File]);
 recompile(File, false, noload, Opts) ->
-   io:format("Recompile: 
~s\n", [File]),
+   % io:format("Recompile: ~ts\n", [File]),
    compile:file(File, [report_errors, report_warnings, error_summary | Opts]);
 recompile(File, false, load, Opts) ->
-   io:format("Recompile: 
~s\n", [File]),
+   % io:format("Recompile: ~ts\n", [File]),
    c:c(File, Opts);
 recompile(File, false, netload, Opts) ->
-   io:format("Recompile: 
~s\n", [File]),
+   % io:format("Recompile: ~ts\n", [File]),
    c:nc(File, Opts).
 
 exists(File) ->
@@ -319,4 +369,4 @@ check_includes2(Epp, File, ObjMTime) ->
          false;
       {error, _Error} ->
          check_includes2(Epp, File, ObjMTime)
-   end.
+   end.