From ccfc1826a368c8c39b5b03064bb81210ce4eeed5 Mon Sep 17 00:00:00 2001 From: Maxim Fedorov Date: Sat, 8 Aug 2020 15:09:11 -0700 Subject: [PATCH] rebar_compiler_epp: use cached results for behaviour/parse_transform This commit makes rebar_compiler_erl to start a gen_server that acts as a filesystem cache, avoiding re-scanning it over and over. --- src/rebar_compiler_epp.erl | 124 +++++++++++++++++++++++++++++++++++++ src/rebar_compiler_erl.erl | 8 ++- 2 files changed, 129 insertions(+), 3 deletions(-) diff --git a/src/rebar_compiler_epp.erl b/src/rebar_compiler_epp.erl index 121e6f31..5334ea15 100644 --- a/src/rebar_compiler_epp.erl +++ b/src/rebar_compiler_epp.erl @@ -4,8 +4,18 @@ %%% @end -module(rebar_compiler_epp). -export([deps/2, resolve_module/2]). +%% cache (a la code path storage, but for dependencies not in code path) +-export([ensure_started/0, flush/0, resolve_source/2]). +-export([init/1, handle_call/3, handle_cast/2]). +%% remove when OTP 19 support is no longer needed +-export([handle_info/2, terminate/2, code_change/3]). + +-behaviour(gen_server). + -include_lib("kernel/include/file.hrl"). +-include("rebar.hrl"). + %%%%%%%%%%%%%%%%%%%%%%%%%%% %%% Basic File Handling %%% %%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -37,6 +47,120 @@ resolve_module(Mod, Paths) -> Path -> {ok, Path} end. +%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%% Cache for deps %%% +%%%%%%%%%%%%%%%%%%%%%%%%%%% +-spec ensure_started() -> ok. +ensure_started() -> + case whereis(?MODULE) of + undefined -> + case gen_server:start({local, ?MODULE}, ?MODULE, [], []) of + {ok, _Pid} -> + ok; + {error, {already_started, _Pid}} -> + ok + end; + Pid when is_pid(Pid) -> + ok + end. + +flush() -> + gen_server:cast(?MODULE, flush). + +%% @doc Resolves "Name" erl module to a path, given list of paths to search. +%% Caches result for subsequent requests. +-spec resolve_source(atom() | file:filename_all(), [file:filename_all()]) -> {true, file:filename_all()} | false. +resolve_source(Name, Dirs) when is_atom(Name) -> + gen_server:call(?MODULE, {resolve, atom_to_list(Name) ++ ".erl", Dirs}); +resolve_source(Name, Dirs) when is_list(Name) -> + gen_server:call(?MODULE, {resolve, Name, Dirs}). + +-record(state, { + %% filesystem cache, denormalised + fs = #{} :: #{file:filename_all() => [file:filename_all()]}, + %% map of module name => abs path + resolved = #{} :: #{file:filename_all() => file:filename_all()} +}). + +init([]) -> + {ok, #state{}}. + +handle_call({resolve, Name, Dirs}, _From, #state{fs = Fs, resolved = Res} = State) -> + case maps:find(Name, Res) of + {ok, Found} -> + {reply, Found, State}; + error -> + {Resolved, NewFs} = resolve(Name, Fs, Dirs), + {reply, Resolved, State#state{resolved = Res#{Name => Resolved}, fs = NewFs}} + end. + +handle_cast(flush, _State) -> + {noreply, #state{}}. + +resolve(_Name, Fs, []) -> + {false, Fs}; +resolve(Name, Fs, [Dir | Tail]) -> + {NewFs, Files} = list_directory(Dir, Fs), + case lists:member(Name, Files) of + true -> + {{true, filename:join(Dir, Name)}, NewFs}; + false -> + resolve(Name, NewFs, Tail) + end. + +%% list_directory/2 caches files in the directory and all subdirectories, +%% to support the behaviour of looking for source files in +%% subdirectories of src/* folder. +%% This may introduce weird dependencies for cases when CT +%% test cases contain test data with files named the same +%% as requested behaviour/parse_transforms, but let's hope +%% it won't happen for many projects. If it does, in fact, +%% it won't cause any damage, just extra unexpected recompiles. +list_directory(Dir, Cache) -> + case maps:find(Dir, Cache) of + {ok, Files} -> + {Cache, Files}; + error -> + case file:list_dir(Dir) of + {ok, DirFiles} -> + %% create a full list of *.erl files under Dir. + {NewFs, Files} = lists:foldl( + fun (File, {DirCache, Files} = Acc) -> + %% recurse into subdirs + FullName = filename:join(Dir, File), + case filelib:is_dir(FullName) of + true -> + {UpdFs, MoreFiles} = list_directory(FullName, DirCache), + {UpdFs, MoreFiles ++ Files}; + false -> + %% ignore all but *.erl files + case filename:extension(File) =:= ".erl" of + true -> + {DirCache, [File | Files]}; + false -> + Acc + end + end + end, + {Cache, []}, DirFiles), + {NewFs#{Dir => Files}, Files}; + {error, Reason} -> + ?DEBUG("Failed to list ~s, ~p", [Dir, Reason]), + {Cache, []} + end + end. + +%%%%%%%%%%%%%%% +%%% OTP 19 %%% +handle_info(_Request, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + %%%%%%%%%%%%%%% %%% PRIVATE %%% %%%%%%%%%%%%%%% diff --git a/src/rebar_compiler_erl.erl b/src/rebar_compiler_erl.erl index 9a8a9377..88783ab0 100644 --- a/src/rebar_compiler_erl.erl +++ b/src/rebar_compiler_erl.erl @@ -100,6 +100,7 @@ dependencies(Source, SourceDir, Dirs) -> end. dependencies(Source, _SourceDir, Dirs, DepOpts) -> + rebar_compiler_epp:ensure_started(), OptPTrans = proplists:get_value(parse_transforms, DepOpts, []), try rebar_compiler_epp:deps(Source, DepOpts) of #{include := AbsIncls, @@ -110,9 +111,9 @@ dependencies(Source, _SourceDir, Dirs, DepOpts) -> %% TODO: check for core transforms? {_MissIncl, _MissInclLib} =/= {[],[]} andalso ?DEBUG("Missing: ~p", [{_MissIncl, _MissInclLib}]), - expand_file_names([module_to_erl(Mod) || Mod <- OptPTrans ++ PTrans], Dirs) ++ - expand_file_names([module_to_erl(Mod) || Mod <- Behaviours], Dirs) ++ - AbsIncls + lists:filtermap( + fun (Mod) -> rebar_compiler_epp:resolve_source(Mod, Dirs) end, + OptPTrans ++ PTrans ++ Behaviours) ++ AbsIncls catch error:{badmatch, {error, Reason}} -> case file:format_error(Reason) of @@ -141,6 +142,7 @@ compile(Source, [{_, OutDir}], Config, ErlOpts) -> end. compile_and_track(Source, [{Ext, OutDir}], Config, ErlOpts) -> + rebar_compiler_epp:flush(), BuildOpts = [{outdir, OutDir} | ErlOpts], Target = target_base(OutDir, Source) ++ Ext, AllOpts = case erlang:function_exported(compile, env_compiler_options, 0) of