From 2e6c2ddc220f229e5aa55e9dc5649935ecab3ca3 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Sun, 26 Jan 2020 15:16:42 -0700 Subject: [PATCH 1/4] git_subdir resource for dependency that is in a subdirectory of repo Because the resource doesn't absolutely require using a sparse checkout, it only matters that rebar3 knows the correct path of the application, the resource is named git_subdir. It has to be git specific because the reason this subdir support is special is the need to keep the code checkout as a usable git clone to run commands against. This means the resource can't just copy the content of the subdirectory to the application out directory. --- rebar.lock | 3 +- src/rebar.app.src | 1 + src/rebar_app_info.erl | 60 ++++++++++++++-------- src/rebar_app_utils.erl | 22 ++++++-- src/rebar_fetch.erl | 7 +-- src/rebar_git_resource.erl | 83 +++++++++++++++++++------------ src/rebar_git_subdir_resource.erl | 77 ++++++++++++++++++++++++++++ 7 files changed, 192 insertions(+), 61 deletions(-) create mode 100644 src/rebar_git_subdir_resource.erl diff --git a/rebar.lock b/rebar.lock index 1fa7dda7..f8146326 100644 --- a/rebar.lock +++ b/rebar.lock @@ -10,7 +10,8 @@ {<<"providers">>,{pkg,<<"providers">>,<<"1.8.1">>},0}, {<<"relx">>, {git,"https://github.com/erlware/relx.git", - {ref,"ac5bb88330e23b29ae0449214af5f62bd16ecdce"}}, + {ref,"ac5bb88330e23b29ae0449214af5f62bd16ecdce"}, + []}, 0}, {<<"ssl_verify_fun">>,{pkg,<<"ssl_verify_fun">>,<<"1.1.5">>},0}]}. [ diff --git a/src/rebar.app.src b/src/rebar.app.src index 40d953bc..303ef27c 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -36,6 +36,7 @@ {log_level, warn}, {resources, [{git, rebar_git_resource}, + {git_subdir, rebar_git_subdir_resource}, {pkg, rebar_pkg_resource}, {hg, rebar_hg_resource}]}, diff --git a/src/rebar_app_info.erl b/src/rebar_app_info.erl index 4b0157f8..3380ef83 100644 --- a/src/rebar_app_info.erl +++ b/src/rebar_app_info.erl @@ -36,6 +36,8 @@ deps/2, dep_level/1, dep_level/2, + fetch_dir/1, + fetch_dir/2, dir/1, dir/2, out_dir/1, @@ -79,29 +81,30 @@ -type project_type() :: rebar3 | mix | undefined. --record(app_info_t, {name :: binary() | undefined, - app_file_src :: file:filename_all() | undefined, - app_file_src_script :: file:filename_all() | undefined, - app_file :: file:filename_all() | undefined, - original_vsn :: binary() | undefined, - parent=root :: binary() | root, - app_details=[] :: list(), - applications=[] :: list(), +-record(app_info_t, {name :: binary() | undefined, + app_file_src :: file:filename_all() | undefined, + app_file_src_script:: file:filename_all() | undefined, + app_file :: file:filename_all() | undefined, + original_vsn :: binary() | undefined, + parent=root :: binary() | root, + app_details=[] :: list(), + applications=[] :: list(), included_applications=[] :: [atom()], - deps=[] :: list(), - profiles=[default] :: [atom()], - default=dict:new() :: rebar_dict(), - opts=dict:new() :: rebar_dict(), - dep_level=0 :: integer(), - dir :: file:name(), - out_dir :: file:name(), - ebin_dir :: file:name(), - source :: string() | tuple() | checkout | undefined, - is_lock=false :: boolean(), - is_checkout=false :: boolean(), - valid :: boolean() | undefined, - project_type :: project_type(), - is_available=false :: boolean()}). + deps=[] :: list(), + profiles=[default] :: [atom()], + default=dict:new() :: rebar_dict(), + opts=dict:new() :: rebar_dict(), + dep_level=0 :: integer(), + fetch_dir :: file:name(), + dir :: file:name(), + out_dir :: file:name(), + ebin_dir :: file:name(), + source :: string() | tuple() | checkout | undefined, + is_lock=false :: boolean(), + is_checkout=false :: boolean(), + valid :: boolean() | undefined, + project_type :: project_type(), + is_available=false :: boolean()}). %%============================================================================ %% types @@ -136,6 +139,7 @@ new(AppName, Vsn) -> new(AppName, Vsn, Dir) -> {ok, #app_info_t{name=rebar_utils:to_binary(AppName), original_vsn=Vsn, + fetch_dir=rebar_utils:to_list(Dir), dir=rebar_utils:to_list(Dir), out_dir=rebar_utils:to_list(Dir), ebin_dir=filename:join(rebar_utils:to_list(Dir), "ebin")}}. @@ -146,6 +150,7 @@ new(AppName, Vsn, Dir) -> new(AppName, Vsn, Dir, Deps) -> {ok, #app_info_t{name=rebar_utils:to_binary(AppName), original_vsn=Vsn, + fetch_dir=rebar_utils:to_list(Dir), dir=rebar_utils:to_list(Dir), out_dir=rebar_utils:to_list(Dir), ebin_dir=filename:join(rebar_utils:to_list(Dir), "ebin"), @@ -158,6 +163,7 @@ new(Parent, AppName, Vsn, Dir, Deps) -> {ok, #app_info_t{name=rebar_utils:to_binary(AppName), parent=Parent, original_vsn=Vsn, + fetch_dir=rebar_utils:to_list(Dir), dir=rebar_utils:to_list(Dir), out_dir=rebar_utils:to_list(Dir), ebin_dir=filename:join(rebar_utils:to_list(Dir), "ebin"), @@ -469,6 +475,16 @@ dep_level(#app_info_t{dep_level=Level}) -> dep_level(AppInfo=#app_info_t{}, Level) -> AppInfo#app_info_t{dep_level=Level}. +%% @doc returns the directory to fetch the dep source to +-spec fetch_dir(t()) -> file:name(). +fetch_dir(#app_info_t{fetch_dir=FetchDir}) -> + FetchDir. + +%% @doc returns the directory to fetch the dep source to +-spec fetch_dir(t(), file:name()) -> file:name(). +fetch_dir(AppInfo=#app_info_t{}, FetchDir) -> + AppInfo#app_info_t{fetch_dir=FetchDir}. + %% @doc returns the directory that contains the app. -spec dir(t()) -> file:name(). dir(#app_info_t{dir=Dir}) -> diff --git a/src/rebar_app_utils.erl b/src/rebar_app_utils.erl index 0b16ae05..65cdd72a 100644 --- a/src/rebar_app_utils.erl +++ b/src/rebar_app_utils.erl @@ -251,13 +251,15 @@ parse_dep(_, Dep, _, _, _) -> Source :: tuple(), IsLock :: boolean(), State :: rebar_state:t(). -dep_to_app(Parent, DepsDir, Name, Vsn, Source, IsLock, State) -> +dep_to_app(Parent, DepsDir, Name, Vsn, Source0, IsLock, State) -> + SubDir = subdir(Source0), + FetchDir = rebar_utils:to_list(filename:join([DepsDir, Name])), CheckoutsDir = rebar_utils:to_list(rebar_dir:checkouts_dir(State, Name)), AppInfo = case rebar_app_info:discover(CheckoutsDir) of {ok, App} -> rebar_app_info:source(rebar_app_info:is_checkout(App, true), checkout); not_found -> - Dir = rebar_utils:to_list(filename:join(DepsDir, Name)), + Dir = rebar_utils:to_list(filename:join([DepsDir, Name, SubDir])), {ok, AppInfo0} = case rebar_app_info:discover(Dir) of {ok, App} -> @@ -267,12 +269,24 @@ dep_to_app(Parent, DepsDir, Name, Vsn, Source, IsLock, State) -> not_found -> rebar_app_info:new(Parent, Name, Vsn, Dir, []) end, - rebar_app_info:source(AppInfo0, Source) + rebar_app_info:source(AppInfo0, Source0) end, Overrides = rebar_app_info:get(AppInfo, overrides, []) ++ rebar_state:get(State, overrides, []), AppInfo2 = rebar_app_info:set(AppInfo, overrides, Overrides), AppInfo5 = rebar_app_info:profiles(AppInfo2, [default]), - rebar_app_info:is_lock(AppInfo5, IsLock). + AppInfo6 = rebar_app_info:fetch_dir(AppInfo5, FetchDir), + rebar_app_info:is_lock(AppInfo6, IsLock). + +%% git has to have special treatment. +%% unlike a resource that simply copies files the git resource needs to be able +%% to access the .git directory and run `git' commands to check against the lock +%% and optionally get the version. Because of this we must keep the clone as a +%% usable git repo clone and using a subdir can not be copied out of it but must +%% have the app info set its directory to be a sub directory of the repo. +subdir({git_subdir, _Repo, _Ref, Dir}) -> + Dir; +subdir(_) -> + "". %% @doc Takes a given application app_info record along with the project. %% If the app is a package, resolve and expand the package definition. diff --git a/src/rebar_fetch.erl b/src/rebar_fetch.erl index 9c76e0ea..0da2a1dd 100644 --- a/src/rebar_fetch.erl +++ b/src/rebar_fetch.erl @@ -54,9 +54,10 @@ download_source_(AppInfo, State) -> ok -> ec_file:mkdir_p(AppDir1), code:del_path(filename:absname(filename:join(AppDir1, "ebin"))), - ok = rebar_file_utils:rm_rf(filename:absname(AppDir1)), - ?DEBUG("Moving checkout ~p to ~p", [TmpDir, filename:absname(AppDir1)]), - rebar_file_utils:mv(TmpDir, filename:absname(AppDir1)); + FetchDir = rebar_app_info:fetch_dir(AppInfo), + ok = rebar_file_utils:rm_rf(filename:absname(FetchDir)), + ?DEBUG("Moving checkout ~p to ~p", [TmpDir, filename:absname(FetchDir)]), + rebar_file_utils:mv(TmpDir, filename:absname(FetchDir)); Error -> Error end. diff --git a/src/rebar_git_resource.erl b/src/rebar_git_resource.erl index e21d9e89..b81d50b3 100644 --- a/src/rebar_git_resource.erl +++ b/src/rebar_git_resource.erl @@ -10,9 +10,15 @@ needs_update/2, make_vsn/2]). +%% for use by rebar_git_sparse_resource +-export([lock_/2, + download_/3, + needs_update_/2, + make_vsn_/1, + git_vsn/0]). + %% For backward compatibilty --export ([ download/3 - ]). +-export ([download/3]). -include("rebar.hrl"). @@ -28,29 +34,31 @@ lock(AppInfo, _) -> check_type_support(), lock_(rebar_app_info:dir(AppInfo), rebar_app_info:source(AppInfo)). -lock_(AppDir, {git, Url, _}) -> - lock_(AppDir, {git, Url}); lock_(AppDir, {git, Url}) -> + lock_(AppDir, {git, Url, [], []}); +lock_(AppDir, {git, Url, Ref}) -> + lock_(AppDir, {git, Url, Ref, []}); +lock_(AppDir, {git, Url, _, Opts}) -> AbortMsg = lists:flatten(io_lib:format("Locking of git dependency failed in ~ts", [AppDir])), Dir = rebar_utils:escape_double_quotes(AppDir), {ok, VsnString} = case os:type() of {win32, _} -> - rebar_utils:sh("git --git-dir=\"" ++ Dir ++ "/.git\" " + rebar_utils:sh("git -C \"" ++ Dir ++ "\" " "--work-tree=\"" ++ Dir ++ "\" rev-parse --verify HEAD", [{use_stdout, false}, {debug_abort_on_error, AbortMsg}]); _ -> - rebar_utils:sh("git --git-dir='" ++ Dir ++ "/.git' rev-parse --verify HEAD", + rebar_utils:sh("git -C '" ++ Dir ++ "' rev-parse --verify HEAD", [{use_stdout, false}, {debug_abort_on_error, AbortMsg}]) end, Ref = rebar_string:trim(VsnString, both, "\n"), - {git, Url, {ref, Ref}}. + {git, Url, {ref, Ref}, Opts}. %% Return true if either the git url or tag/branch/ref is not the same as the currently %% checked out git repo for the dep needs_update(AppInfo, _) -> check_type_support(), - needs_update_(rebar_app_info:dir(AppInfo), rebar_app_info:source(AppInfo)). + needs_update_(rebar_app_info:dir(AppInfo), rm_source_opts(rebar_app_info:source(AppInfo))). needs_update_(Dir, {git, Url, {tag, Tag}}) -> {ok, Current} = rebar_utils:sh(?FMT("git describe --tags --exact-match", []), @@ -120,6 +128,8 @@ parse_git_url(not_scp, Url) -> download(TmpDir, AppInfo, State, _) -> check_type_support(), case download_(TmpDir, rebar_app_info:source(AppInfo), State) of + ok -> + ok; {ok, _} -> ok; {error, Reason} -> @@ -128,29 +138,37 @@ download(TmpDir, AppInfo, State, _) -> {error, Error} end. +rm_source_opts({Type, Url, Ref}) -> + {Type, Url, Ref}; +rm_source_opts({Type, Url, Ref, _Opts}) -> + {Type, Url, Ref}. + %% For backward compatibilty download(Dir, AppInfo, State) -> download_(Dir, AppInfo, State). download_(Dir, {git, Url}, State) -> ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []), - download_(Dir, {git, Url, {branch, "master"}}, State); + download_(Dir, {git, Url, {branch, "master"}, []}, State); download_(Dir, {git, Url, ""}, State) -> ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []), - download_(Dir, {git, Url, {branch, "master"}}, State); -download_(Dir, {git, Url, {branch, Branch}}, _State) -> + download_(Dir, {git, Url, {branch, "master"}, []}, State); +download_(Dir, {git, Url, Checkout}, _State) -> + %% add an empty opts element to the tupel if there is none + download_(Dir, {git, Url, Checkout, []}, _State); +download_(Dir, {git, Url, {branch, Branch}, _Opts}, _State) -> ok = filelib:ensure_dir(Dir), maybe_warn_local_url(Url), git_clone(branch, git_vsn(), Url, Dir, Branch); -download_(Dir, {git, Url, {tag, Tag}}, _State) -> +download_(Dir, {git, Url, {tag, Tag}, _Opts}, _State) -> ok = filelib:ensure_dir(Dir), maybe_warn_local_url(Url), git_clone(tag, git_vsn(), Url, Dir, Tag); -download_(Dir, {git, Url, {ref, Ref}}, _State) -> +download_(Dir, {git, Url, {ref, Ref}, _Opts}, _State) -> ok = filelib:ensure_dir(Dir), maybe_warn_local_url(Url), git_clone(ref, git_vsn(), Url, Dir, Ref); -download_(Dir, {git, Url, Rev}, _State) -> +download_(Dir, {git, Url, Rev, _Opts}, _State) -> ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []), ok = filelib:ensure_dir(Dir), maybe_warn_local_url(Url), @@ -167,57 +185,60 @@ maybe_warn_local_url(Url) -> end. %% Use different git clone commands depending on git --version -git_clone(branch,Vsn,Url,Dir,Branch) when Vsn >= {1,7,10}; Vsn =:= undefined -> +git_clone(branch, GitVsn, Url, Dir, Branch) when GitVsn >= {1,7,10}; GitVsn =:= undefined -> rebar_utils:sh(?FMT("git clone ~ts ~ts ~ts -b ~ts --single-branch", [git_clone_options(), rebar_utils:escape_chars(Url), rebar_utils:escape_chars(filename:basename(Dir)), rebar_utils:escape_chars(Branch)]), - [{cd, filename:dirname(Dir)}]); -git_clone(branch,_Vsn,Url,Dir,Branch) -> + [{cd, filename:dirname(Dir)}]), + ok; +git_clone(branch, _GitVsn, Url, Dir, Branch) -> rebar_utils:sh(?FMT("git clone ~ts ~ts ~ts -b ~ts", [git_clone_options(), rebar_utils:escape_chars(Url), rebar_utils:escape_chars(filename:basename(Dir)), rebar_utils:escape_chars(Branch)]), - [{cd, filename:dirname(Dir)}]); -git_clone(tag,Vsn,Url,Dir,Tag) when Vsn >= {1,7,10}; Vsn =:= undefined -> + [{cd, filename:dirname(Dir)}]), + ok; +git_clone(tag, GitVsn, Url, Dir, Tag) when GitVsn >= {1,7,10}; GitVsn =:= undefined -> rebar_utils:sh(?FMT("git clone ~ts ~ts ~ts -b ~ts --single-branch", [git_clone_options(), rebar_utils:escape_chars(Url), rebar_utils:escape_chars(filename:basename(Dir)), rebar_utils:escape_chars(Tag)]), - [{cd, filename:dirname(Dir)}]); -git_clone(tag,_Vsn,Url,Dir,Tag) -> + [{cd, filename:dirname(Dir)}]), + ok; +git_clone(tag, _GitVsn, Url, Dir,Tag) -> rebar_utils:sh(?FMT("git clone ~ts ~ts ~ts -b ~ts", [git_clone_options(), rebar_utils:escape_chars(Url), rebar_utils:escape_chars(filename:basename(Dir)), rebar_utils:escape_chars(Tag)]), - [{cd, filename:dirname(Dir)}]); -git_clone(ref,_Vsn,Url,Dir,Ref) -> + [{cd, filename:dirname(Dir)}]), + ok; +git_clone(ref, _GitVsn, Url, Dir, Ref) -> rebar_utils:sh(?FMT("git clone ~ts -n ~ts ~ts", [git_clone_options(), rebar_utils:escape_chars(Url), rebar_utils:escape_chars(filename:basename(Dir))]), [{cd, filename:dirname(Dir)}]), - rebar_utils:sh(?FMT("git checkout -q ~ts", [Ref]), [{cd, Dir}]); -git_clone(rev,_Vsn,Url,Dir,Rev) -> + rebar_utils:sh(?FMT("git checkout -q ~ts", [rebar_utils:escape_chars(Ref)]), [{cd, Dir}]); +git_clone(rev, _GitVsn, Url, Dir, Rev) -> rebar_utils:sh(?FMT("git clone ~ts -n ~ts ~ts", [git_clone_options(), rebar_utils:escape_chars(Url), rebar_utils:escape_chars(filename:basename(Dir))]), [{cd, filename:dirname(Dir)}]), - rebar_utils:sh(?FMT("git checkout -q ~ts", [rebar_utils:escape_chars(Rev)]), - [{cd, Dir}]). + rebar_utils:sh(?FMT("git checkout -q ~ts", [rebar_utils:escape_chars(Rev)]), [{cd, Dir}]). git_vsn() -> case application:get_env(rebar, git_vsn) of - {ok, Vsn} -> Vsn; + {ok, GitVsn} -> GitVsn; undefined -> - Vsn = git_vsn_fetch(), - application:set_env(rebar, git_vsn, Vsn), - Vsn + GitVsn = git_vsn_fetch(), + application:set_env(rebar, git_vsn, GitVsn), + GitVsn end. git_vsn_fetch() -> diff --git a/src/rebar_git_subdir_resource.erl b/src/rebar_git_subdir_resource.erl new file mode 100644 index 00000000..65c588d8 --- /dev/null +++ b/src/rebar_git_subdir_resource.erl @@ -0,0 +1,77 @@ +%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ts=4 sw=4 et +-module(rebar_git_subdir_resource). + +-behaviour(rebar_resource_v2). + +-export([init/2, + lock/2, + download/4, + needs_update/2, + make_vsn/2]). + +-include("rebar.hrl"). + +%% Regex used for parsing scp style remote url +-define(SCP_PATTERN, "\\A(?[^@]+)@(?[^:]+):(?.+)\\z"). + +-spec init(atom(), rebar_state:t()) -> {ok, rebar_resource_v2:resource()}. +init(Type, _State) -> + Resource = rebar_resource_v2:new(Type, ?MODULE, #{}), + {ok, Resource}. + +lock(AppInfo, _) -> + {git_subdir, Url, Checkout, Dir} = rebar_app_info:source(AppInfo), + {git, Url1, {ref, Ref}, _Opts} = + rebar_git_resource:lock_(rebar_app_info:dir(AppInfo), {git, Url, Checkout}), + {git_subdir, Url1, {ref, Ref}, Dir}. + +download(TmpDir, AppInfo, State, _) -> + {git_subdir, Url, Checkout, SparseDir} = rebar_app_info:source(AppInfo), + case rebar_git_resource:download_(TmpDir, {git, Url, Checkout}, State) of + ok -> + sparse_checkout(rebar_git_resource:git_vsn(), TmpDir, to_ref(Checkout), SparseDir); + {ok, _} -> + sparse_checkout(rebar_git_resource:git_vsn(), TmpDir, to_ref(Checkout), SparseDir); + {error, Reason} -> + {error, Reason}; + Error -> + {error, Error} + end. + +%% Return true if either the git url or tag/branch/ref is not the same as the currently +%% checked out git repo for the dep +needs_update(AppInfo, _) -> + {git_subdir, Url, Ref, _Dir} = rebar_app_info:source(AppInfo), + rebar_git_resource:needs_update_(rebar_app_info:dir(AppInfo), {git, Url, Ref}). + +make_vsn(AppInfo, _) -> + Dir = rebar_app_info:dir(AppInfo), + rebar_git_resource:make_vsn_(Dir). + +%% + +to_ref({branch, Branch}) -> + Branch; +to_ref({tag, Tag}) -> + Tag; +to_ref({ref, Ref}) -> + Ref; +to_ref(Rev) -> + Rev. + +sparse_checkout(GitVsn, Dir, Ref, SparseDir) when GitVsn >= {1,7,4}; + GitVsn =:= undefined -> + ?DEBUG("doing sparse checkout in ~s of dir ~s", [Dir, SparseDir]), + + rebar_utils:sh(?FMT("git --git-dir=.git config core.sparsecheckout true", []), + [{cd, Dir}]), + filelib:ensure_dir(filename:join(Dir, ".git/info/sparse-checkout")), + file:write_file(filename:join(Dir, ".git/info/sparse-checkout"), SparseDir), + rebar_utils:sh(?FMT("git checkout -q ~ts", [rebar_utils:escape_chars(Ref)]), [{cd, Dir}]), + ok; +sparse_checkout(_, _, _, _) -> + %% sparse checkout not supported but we can still use the subdirectory + %% so no need to fail, just don't do the sparse checkout + ?DEBUG("too old a git version to do a sparse checkout for a subdir dep", []), + ok. From bff3b76dba9d87e571024dfc95e7a9965a866eee Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Mon, 4 May 2020 09:20:59 -0600 Subject: [PATCH 2/4] very quick test of git_subdir --- test/mock_git_subdir_resource.erl | 142 ++++++++++++++++++++++++++++++ test/rebar_compile_SUITE.erl | 18 +++- test/rebar_test_utils.erl | 31 ++++++- 3 files changed, 189 insertions(+), 2 deletions(-) create mode 100644 test/mock_git_subdir_resource.erl diff --git a/test/mock_git_subdir_resource.erl b/test/mock_git_subdir_resource.erl new file mode 100644 index 00000000..7c1c6e9c --- /dev/null +++ b/test/mock_git_subdir_resource.erl @@ -0,0 +1,142 @@ +%%% Mock a git_subdir resource and create an app magically for each +%%% URL and subdirectory submitted. +-module(mock_git_subdir_resource). +-export([mock/0, mock/1, mock/2, unmock/0]). +-define(MOD, rebar_git_subdir_resource). + +%%%%%%%%%%%%%%%%% +%%% Interface %%% +%%%%%%%%%%%%%%%%% + +%% @doc same as `mock([])'. +mock() -> mock([]). + +%% @doc Mocks a fake version of the git resource fetcher that creates +%% empty applications magically, rather than trying to download them. +%% Specific config options are explained in each of the private functions. +-spec mock(Opts) -> ok when + Opts :: [Option], + Option :: {update, [App]} + | {default_vsn, Vsn} + | {override_vsn, [{App, Vsn}]} + | {deps, [{App, [Dep]}]}, + App :: string(), + Dep :: {App, {git_subdir, string(), term(), string()}} + | {pkg, App, term()}, + Vsn :: string(). +mock(Opts) -> + mock(Opts, create_app). + +mock(Opts, CreateType) -> + meck:new(?MOD, [no_link, passthrough]), + mock_lock(Opts), + mock_update(Opts), + mock_vsn(Opts), + mock_download(Opts, CreateType), + ok. + +unmock() -> + meck:unload(?MOD). + +%%%%%%%%%%%%%%% +%%% Private %%% +%%%%%%%%%%%%%%% + +%% @doc creates values for a lock file. The refs are fake, but +%% tags and existing refs declared for a dependency are preserved. +mock_lock(_) -> + meck:expect( + ?MOD, lock, + fun(AppInfo, _) -> + case rebar_app_info:source(AppInfo) of + {git_subdir, Url, {tag, Ref}, Dir} -> {git_subdir, Url, {ref, Ref}, Dir}; + {git_subdir, Url, {ref, Ref}, Dir} -> {git_subdir, Url, {ref, Ref}, Dir}; + {git_subdir, Url, Dir} -> {git_subdir, Url, {ref, "0.0.0"}, Dir}; + {git_subdir, Url, _, Dir} -> {git_subdir, Url, {ref, "0.0.0"}, Dir} + end + end). + +%% @doc The config passed to the `mock/2' function can specify which apps +%% should be updated on a per-name basis: `{update, ["App1", "App3"]}'. +mock_update(Opts) -> + ToUpdate = proplists:get_value(upgrade, Opts, []), +% ct:pal("TOUp: ~p", [ToUpdate]), + meck:expect( + ?MOD, needs_update, + fun(AppInfo, _) -> + {git_subdir, Url, _Ref} = rebar_app_info:source(AppInfo), + App = app(Url), +% ct:pal("Needed update? ~p (~p) -> ~p", [App, {Url,_Ref}, lists:member(App, ToUpdate)]), + lists:member(App, ToUpdate) + end). + +%% @doc Tries to fetch a version from the `*.app.src' file or otherwise +%% just returns random stuff, avoiding to check for the presence of git. +%% This probably breaks the assumption that stable references are returned. +%% +%% This function can't respect the `override_vsn' option because if the +%% .app.src file isn't there, we can't find the app name either. +mock_vsn(Opts) -> + Default = proplists:get_value(default_vsn, Opts, "0.0.0"), + meck:expect( + ?MOD, make_vsn, + fun(AppInfo, _) -> + Dir = rebar_app_info:dir(AppInfo), + case filelib:wildcard("*.app.src", filename:join([Dir,"src"])) of + [AppSrc] -> + {ok, App} = file:consult(AppSrc), + Vsn = proplists:get_value(vsn, App), + {plain, Vsn}; + _ -> + {plain, Default} + end + end). + +%% @doc For each app to download, create a dummy app on disk instead. +%% The configuration for this one (passed in from `mock/1') includes: +%% +%% - Specify a version, branch, ref, or tag via the `{git_subdir, URL, {_, Vsn}' +%% format to specify a path. +%% - If there is no version submitted (`{git_subdir, URL}'), the function instead +%% reads from the `override_vsn' proplist (`{override_vsn, {"App1","1.2.3"}'), +%% and otherwise uses the value associated with `default_vsn'. +%% - Dependencies for each application must be passed of the form: +%% `{deps, [{"app1", [{app2, ".*", {git_subdir, ...}}]}]}' -- basically +%% the `deps' option takes a key/value list of terms to output directly +%% into a `rebar.config' file to describe dependencies. +mock_download(Opts, CreateType) -> + Deps = proplists:get_value(deps, Opts, []), + Config = proplists:get_value(config, Opts, []), + Default = proplists:get_value(default_vsn, Opts, "0.0.0"), + Overrides = proplists:get_value(override_vsn, Opts, []), + meck:expect( + ?MOD, download, + fun (Dir, AppInfo, _, _) -> + Git = rebar_app_info:source(AppInfo), + filelib:ensure_dir(Dir), + {git_subdir, Url, {_, Vsn}, SubDir} = normalize_git(Git, Overrides, Default), + FullSubDir = filename:join(Dir, SubDir), + filelib:ensure_dir(FullSubDir), + App = app(Url), + AppDeps = proplists:get_value({App,Vsn}, Deps, []), + rebar_test_utils:CreateType( + FullSubDir, App, Vsn, + [kernel, stdlib] ++ [element(1,D) || D <- AppDeps] + ), + rebar_test_utils:create_config(FullSubDir, [{deps, AppDeps}]++Config), + ok + end). + +%%%%%%%%%%%%%%% +%%% Helpers %%% +%%%%%%%%%%%%%%% +app(Path) -> + filename:basename(Path, ".git"). + +normalize_git({git_subdir, Url, SubDir}, Overrides, Default) -> + Vsn = proplists:get_value(app(Url), Overrides, Default), + {git, Url, {tag, Vsn}, SubDir}; +normalize_git({git_subdir, Url, Branch, SubDir}, _, _) when is_list(Branch) -> + {git, Url, {branch, Branch}, SubDir}; +normalize_git(Git, _, _) -> + Git. diff --git a/test/rebar_compile_SUITE.erl b/test/rebar_compile_SUITE.erl index 55e0cda5..3120bfed 100644 --- a/test/rebar_compile_SUITE.erl +++ b/test/rebar_compile_SUITE.erl @@ -27,7 +27,7 @@ all() -> parse_transform_test, erl_first_files_test, mib_test, umbrella_mib_first_test, only_default_transitive_deps, clean_all, clean_specific, profile_deps, deps_build_in_prod, only_deps, - override_deps, override_add_deps, override_del_deps, + override_deps, git_subdir_deps, override_add_deps, override_del_deps, override_opts, override_add_opts, override_del_opts, apply_overrides_exactly_once, override_only_deps, profile_override_deps, profile_override_add_deps, profile_override_del_deps, @@ -1531,6 +1531,22 @@ override_deps(Config) -> {dep_not_exist, "other_dep"}]} ). +git_subdir_deps(Config) -> + Deps = rebar_test_utils:expand_deps(git_subdir, [{"some_dep", "0.0.1", [{"other_dep", "0.0.1", []}]}]), + TopDeps = rebar_test_utils:top_level_deps(Deps), + + {SrcDeps, _} = rebar_test_utils:flat_deps(Deps), + mock_git_subdir_resource:mock([{deps, SrcDeps}]), + + RebarConfig = [ + {deps, TopDeps} + ], + rebar_test_utils:run_and_check( + Config, RebarConfig, ["compile"], + {ok, [{subdir_dep, "some_dep"}, + {subdir_dep, "other_dep"}]} + ). + override_add_deps(Config) -> Deps = rebar_test_utils:expand_deps(git, [{"some_dep", "0.0.1", [{"other_dep", "0.0.1", []}]}]), TopDeps = rebar_test_utils:top_level_deps(Deps), diff --git a/test/rebar_test_utils.erl b/test/rebar_test_utils.erl index 0f5948d2..b93403ab 100644 --- a/test/rebar_test_utils.erl +++ b/test/rebar_test_utils.erl @@ -183,6 +183,12 @@ random_seed() -> -endif. expand_deps(_, []) -> []; +expand_deps(git_subdir, [{Name, Deps} | Rest]) -> + Dep = {Name, {git_subdir, "https://example.org/user/"++Name++".git", {branch, "master"}, "appsubdir"}}, + [{Dep, expand_deps(git_subdir, Deps)} | expand_deps(git_subdir, Rest)]; +expand_deps(git_subdir, [{Name, Vsn, Deps} | Rest]) -> + Dep = {Name, Vsn, {git_subdir, "https://example.org/user/"++Name++".git", {tag, Vsn}, "appsubdir"}}, + [{Dep, expand_deps(git_subdir, Deps)} | expand_deps(git_subdir, Rest)]; expand_deps(git, [{Name, Deps} | Rest]) -> Dep = {Name, ".*", {git, "https://example.org/user/"++Name++".git", "master"}}, [{Dep, expand_deps(git, Deps)} | expand_deps(git, Rest)]; @@ -226,6 +232,12 @@ flat_deps([{{pkg, Name, Vsn, undefined, undefined}, PkgDeps} | Rest], Src, Pkg) Src, Pkg ++ [Current | FlatPkgDeps]); flat_deps([{{Name,_Vsn,Ref}, Deps} | Rest], Src, Pkg) -> + Current = {{Name,vsn_from_ref(Ref)}, top_level_deps(Deps)}, + {FlatDeps, FlatPkgDeps} = flat_deps(Deps), + flat_deps(Rest, + Src ++ [Current | FlatDeps], + Pkg ++ FlatPkgDeps); +flat_deps([{{Name,Ref}, Deps} | Rest], Src, Pkg) -> Current = {{Name,vsn_from_ref(Ref)}, top_level_deps(Deps)}, {FlatDeps, FlatPkgDeps} = flat_deps(Deps), flat_deps(Rest, @@ -233,7 +245,9 @@ flat_deps([{{Name,_Vsn,Ref}, Deps} | Rest], Src, Pkg) -> Pkg ++ FlatPkgDeps). vsn_from_ref({git, _, {_, Vsn}}) -> Vsn; -vsn_from_ref({git, _, Vsn}) -> Vsn. +vsn_from_ref({git, _, Vsn}) -> Vsn; +vsn_from_ref({git_subdir, _, {_, Vsn}, _}) -> Vsn; +vsn_from_ref({git_subdir, _, Vsn, _}) -> Vsn. top_level_deps([]) -> []; top_level_deps([{{pkg, Name, Vsn, undefined, undefined}, _} | Deps]) -> @@ -246,6 +260,7 @@ top_level_deps([{{Name, Vsn, Ref}, _} | Deps]) -> %%%%%%%%%%%%%%% check_results(AppDir, Expected, ProfileRun) -> BuildDirs = filelib:wildcard(filename:join([AppDir, "_build", ProfileRun, "lib", "*"])), + BuildSubDirs = filelib:wildcard(filename:join([AppDir, "_build", ProfileRun, "lib", "*", "*"])), PluginDirs = filelib:wildcard(filename:join([AppDir, "_build", ProfileRun, "plugins", "*"])), GlobalPluginDirs = filelib:wildcard(filename:join([AppDir, "global", "plugins", "*"])), CheckoutsDirs = filelib:wildcard(filename:join([AppDir, "_checkouts", "*"])), @@ -259,7 +274,9 @@ check_results(AppDir, Expected, ProfileRun) -> ValidDepsNames = [{ec_cnv:to_list(rebar_app_info:name(App)), App} || App <- ValidApps], Deps = rebar_app_discover:find_apps(BuildDirs, all), + SubDeps = rebar_app_discover:find_apps(BuildSubDirs, all), DepsNames = [{ec_cnv:to_list(rebar_app_info:name(App)), App} || App <- Deps], + SubDirDepsNames = [{ec_cnv:to_list(rebar_app_info:name(App)), App} || App <- SubDeps], Checkouts = rebar_app_discover:find_apps(CheckoutsDirs, all), CheckoutsNames = [{ec_cnv:to_list(rebar_app_info:name(App)), App} || App <- Checkouts], Plugins = rebar_app_discover:find_apps(PluginDirs, all), @@ -323,6 +340,18 @@ check_results(AppDir, Expected, ProfileRun) -> ?assertEqual(iolist_to_binary(Vsn), iolist_to_binary(rebar_app_info:original_vsn(App))) end + ; ({subdir_dep, Name}) -> + ct:pal("Subdir Dep Name: ~p", [Name]), + ?assertNotEqual(false, lists:keyfind(Name, 1, SubDirDepsNames)) + ; ({subdir_dep, Name, Vsn}) -> + ct:pal("Subdir Dep Name: ~p, Vsn: ~p", [Name, Vsn]), + case lists:keyfind(Name, 1, SubDirDepsNames) of + false -> + error({dep_not_found, Name}); + {Name, App} -> + ?assertEqual(iolist_to_binary(Vsn), + iolist_to_binary(rebar_app_info:original_vsn(App))) + end ; ({plugin, Name}) -> ct:pal("Plugin Name: ~p", [Name]), ?assertNotEqual(false, lists:keyfind(Name, 1, PluginsNames)) From f0c8c202afd5b733b7fa11044d06d163e5a39485 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Mon, 4 May 2020 09:46:26 -0600 Subject: [PATCH 3/4] wip: rework rebar_prv_path to use app info --- src/rebar_prv_path.erl | 33 +++++++++++---------------------- 1 file changed, 11 insertions(+), 22 deletions(-) diff --git a/src/rebar_prv_path.erl b/src/rebar_prv_path.erl index 5374b0c1..4562db8f 100644 --- a/src/rebar_prv_path.erl +++ b/src/rebar_prv_path.erl @@ -12,7 +12,7 @@ -include("rebar.hrl"). -define(PROVIDER, path). --define(DEPS, [app_discovery]). +-define(DEPS, [compile]). %% =================================================================== %% Public API @@ -54,11 +54,11 @@ filter_apps(RawOpts, State) -> [] -> ProjectDeps = project_deps(State), ProjectApps = rebar_state:project_apps(State), - lists:map(fun(A) -> binary_to_list(rebar_app_info:name(A)) end, ProjectApps) ++ ProjectDeps; - _ -> Apps + ProjectApps ++ ProjectDeps; + _ -> + Apps end. - paths([], _, State, Acc) -> print_paths_if_exist(lists:reverse(Acc), State); paths([{base, true}|Rest], Apps, State, Acc) -> paths(Rest, Apps, State, [base_dir(State)|Acc]); @@ -80,12 +80,12 @@ bin_dir(State) -> io_lib:format("~ts/bin", [rebar_dir:base_dir(State)]). lib_dir(State) -> io_lib:format("~ts", [rebar_dir:deps_dir(State)]). rel_dir(State) -> io_lib:format("~ts/rel", [rebar_dir:base_dir(State)]). -ebin_dirs(Apps, State) -> - lists:map(fun(App) -> io_lib:format("~ts/~ts/ebin", [rebar_dir:deps_dir(State), App]) end, Apps). -priv_dirs(Apps, State) -> - lists:map(fun(App) -> io_lib:format("~ts/~ts/priv", [rebar_dir:deps_dir(State), App]) end, Apps). -src_dirs(Apps, State) -> - lists:map(fun(App) -> io_lib:format("~ts/~ts/src", [rebar_dir:deps_dir(State), App]) end, Apps). +ebin_dirs(Apps, _State) -> + lists:map(fun(App) -> rebar_app_info:ebin_dir(App) end, Apps). +priv_dirs(Apps, _State) -> + lists:map(fun(App) -> rebar_app_info:priv_dir(App) end, Apps). +src_dirs(Apps, _State) -> + lists:map(fun(App) -> filename:join(rebar_app_info:out_dir(App), "src") end, Apps). print_paths_if_exist(Paths, State) -> {RawOpts, _} = rebar_state:command_parsed_args(State), @@ -94,18 +94,7 @@ print_paths_if_exist(Paths, State) -> io:format("~ts", [rebar_string:join(RealPaths, Sep)]). project_deps(State) -> - Profiles = rebar_state:current_profiles(State), - DepList = lists:foldl(fun(Profile, Acc) -> rebar_state:get(State, {deps, Profile}, []) ++ Acc end, [], Profiles), - LockList = lists:foldl(fun(Profile, Acc) -> rebar_state:get(State, {locks, Profile}, []) ++ Acc end, [], Profiles), - Deps = [normalize(name(Dep)) || Dep <- DepList++LockList], - lists:usort(Deps). - -name(App) when is_tuple(App) -> element(1, App); -name(Name) when is_binary(Name); is_list(Name); is_atom(Name) -> Name. - -normalize(AppName) when is_list(AppName) -> AppName; -normalize(AppName) when is_atom(AppName) -> atom_to_list(AppName); -normalize(AppName) when is_binary(AppName) -> binary_to_list(AppName). + rebar_state:all_deps(State). path_opts(_State) -> [{app, undefined, "app", string, help(app)}, From b3fad9ac0ceefb952b1fcf091b1544dd9b54dee7 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Wed, 6 May 2020 09:30:24 -0600 Subject: [PATCH 4/4] filter out non-directories from app discover search --- src/rebar_app_discover.erl | 9 +++++++-- test/rebar_test_utils.erl | 3 ++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/rebar_app_discover.erl b/src/rebar_app_discover.erl index ca837c54..0a5bede5 100644 --- a/src/rebar_app_discover.erl +++ b/src/rebar_app_discover.erl @@ -216,8 +216,13 @@ reset_hooks(Opts, CurrentProfiles) -> -spec all_app_dirs([file:name()]) -> [{file:name(), [file:name()]}]. all_app_dirs(LibDirs) -> lists:flatmap(fun(LibDir) -> - {_, SrcDirs} = find_config_src(LibDir, ["src"]), - app_dirs(LibDir, SrcDirs) + case filelib:is_dir(LibDir) of + true -> + {_, SrcDirs} = find_config_src(LibDir, ["src"]), + app_dirs(LibDir, SrcDirs); + false -> + [] + end end, LibDirs). %% @private find the directories for all apps based on their source dirs diff --git a/test/rebar_test_utils.erl b/test/rebar_test_utils.erl index b93403ab..99547d66 100644 --- a/test/rebar_test_utils.erl +++ b/test/rebar_test_utils.erl @@ -260,7 +260,8 @@ top_level_deps([{{Name, Vsn, Ref}, _} | Deps]) -> %%%%%%%%%%%%%%% check_results(AppDir, Expected, ProfileRun) -> BuildDirs = filelib:wildcard(filename:join([AppDir, "_build", ProfileRun, "lib", "*"])), - BuildSubDirs = filelib:wildcard(filename:join([AppDir, "_build", ProfileRun, "lib", "*", "*"])), + BuildSubDirs = [D || D <- filelib:wildcard(filename:join([AppDir, "_build", ProfileRun, "lib", "*", "*"])), + filelib:is_dir(D)], PluginDirs = filelib:wildcard(filename:join([AppDir, "_build", ProfileRun, "plugins", "*"])), GlobalPluginDirs = filelib:wildcard(filename:join([AppDir, "global", "plugins", "*"])), CheckoutsDirs = filelib:wildcard(filename:join([AppDir, "_checkouts", "*"])),