diff --git a/rebar.lock b/rebar.lock index 1fa7dda7..f8146326 100644 --- a/rebar.lock +++ b/rebar.lock @@ -10,7 +10,8 @@ {<<"providers">>,{pkg,<<"providers">>,<<"1.8.1">>},0}, {<<"relx">>, {git,"https://github.com/erlware/relx.git", - {ref,"ac5bb88330e23b29ae0449214af5f62bd16ecdce"}}, + {ref,"ac5bb88330e23b29ae0449214af5f62bd16ecdce"}, + []}, 0}, {<<"ssl_verify_fun">>,{pkg,<<"ssl_verify_fun">>,<<"1.1.5">>},0}]}. [ diff --git a/src/rebar.app.src b/src/rebar.app.src index 40d953bc..303ef27c 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -36,6 +36,7 @@ {log_level, warn}, {resources, [{git, rebar_git_resource}, + {git_subdir, rebar_git_subdir_resource}, {pkg, rebar_pkg_resource}, {hg, rebar_hg_resource}]}, diff --git a/src/rebar_app_info.erl b/src/rebar_app_info.erl index 4b0157f8..3380ef83 100644 --- a/src/rebar_app_info.erl +++ b/src/rebar_app_info.erl @@ -36,6 +36,8 @@ deps/2, dep_level/1, dep_level/2, + fetch_dir/1, + fetch_dir/2, dir/1, dir/2, out_dir/1, @@ -79,29 +81,30 @@ -type project_type() :: rebar3 | mix | undefined. --record(app_info_t, {name :: binary() | undefined, - app_file_src :: file:filename_all() | undefined, - app_file_src_script :: file:filename_all() | undefined, - app_file :: file:filename_all() | undefined, - original_vsn :: binary() | undefined, - parent=root :: binary() | root, - app_details=[] :: list(), - applications=[] :: list(), +-record(app_info_t, {name :: binary() | undefined, + app_file_src :: file:filename_all() | undefined, + app_file_src_script:: file:filename_all() | undefined, + app_file :: file:filename_all() | undefined, + original_vsn :: binary() | undefined, + parent=root :: binary() | root, + app_details=[] :: list(), + applications=[] :: list(), included_applications=[] :: [atom()], - deps=[] :: list(), - profiles=[default] :: [atom()], - default=dict:new() :: rebar_dict(), - opts=dict:new() :: rebar_dict(), - dep_level=0 :: integer(), - dir :: file:name(), - out_dir :: file:name(), - ebin_dir :: file:name(), - source :: string() | tuple() | checkout | undefined, - is_lock=false :: boolean(), - is_checkout=false :: boolean(), - valid :: boolean() | undefined, - project_type :: project_type(), - is_available=false :: boolean()}). + deps=[] :: list(), + profiles=[default] :: [atom()], + default=dict:new() :: rebar_dict(), + opts=dict:new() :: rebar_dict(), + dep_level=0 :: integer(), + fetch_dir :: file:name(), + dir :: file:name(), + out_dir :: file:name(), + ebin_dir :: file:name(), + source :: string() | tuple() | checkout | undefined, + is_lock=false :: boolean(), + is_checkout=false :: boolean(), + valid :: boolean() | undefined, + project_type :: project_type(), + is_available=false :: boolean()}). %%============================================================================ %% types @@ -136,6 +139,7 @@ new(AppName, Vsn) -> new(AppName, Vsn, Dir) -> {ok, #app_info_t{name=rebar_utils:to_binary(AppName), original_vsn=Vsn, + fetch_dir=rebar_utils:to_list(Dir), dir=rebar_utils:to_list(Dir), out_dir=rebar_utils:to_list(Dir), ebin_dir=filename:join(rebar_utils:to_list(Dir), "ebin")}}. @@ -146,6 +150,7 @@ new(AppName, Vsn, Dir) -> new(AppName, Vsn, Dir, Deps) -> {ok, #app_info_t{name=rebar_utils:to_binary(AppName), original_vsn=Vsn, + fetch_dir=rebar_utils:to_list(Dir), dir=rebar_utils:to_list(Dir), out_dir=rebar_utils:to_list(Dir), ebin_dir=filename:join(rebar_utils:to_list(Dir), "ebin"), @@ -158,6 +163,7 @@ new(Parent, AppName, Vsn, Dir, Deps) -> {ok, #app_info_t{name=rebar_utils:to_binary(AppName), parent=Parent, original_vsn=Vsn, + fetch_dir=rebar_utils:to_list(Dir), dir=rebar_utils:to_list(Dir), out_dir=rebar_utils:to_list(Dir), ebin_dir=filename:join(rebar_utils:to_list(Dir), "ebin"), @@ -469,6 +475,16 @@ dep_level(#app_info_t{dep_level=Level}) -> dep_level(AppInfo=#app_info_t{}, Level) -> AppInfo#app_info_t{dep_level=Level}. +%% @doc returns the directory to fetch the dep source to +-spec fetch_dir(t()) -> file:name(). +fetch_dir(#app_info_t{fetch_dir=FetchDir}) -> + FetchDir. + +%% @doc returns the directory to fetch the dep source to +-spec fetch_dir(t(), file:name()) -> file:name(). +fetch_dir(AppInfo=#app_info_t{}, FetchDir) -> + AppInfo#app_info_t{fetch_dir=FetchDir}. + %% @doc returns the directory that contains the app. -spec dir(t()) -> file:name(). dir(#app_info_t{dir=Dir}) -> diff --git a/src/rebar_app_utils.erl b/src/rebar_app_utils.erl index 0b16ae05..65cdd72a 100644 --- a/src/rebar_app_utils.erl +++ b/src/rebar_app_utils.erl @@ -251,13 +251,15 @@ parse_dep(_, Dep, _, _, _) -> Source :: tuple(), IsLock :: boolean(), State :: rebar_state:t(). -dep_to_app(Parent, DepsDir, Name, Vsn, Source, IsLock, State) -> +dep_to_app(Parent, DepsDir, Name, Vsn, Source0, IsLock, State) -> + SubDir = subdir(Source0), + FetchDir = rebar_utils:to_list(filename:join([DepsDir, Name])), CheckoutsDir = rebar_utils:to_list(rebar_dir:checkouts_dir(State, Name)), AppInfo = case rebar_app_info:discover(CheckoutsDir) of {ok, App} -> rebar_app_info:source(rebar_app_info:is_checkout(App, true), checkout); not_found -> - Dir = rebar_utils:to_list(filename:join(DepsDir, Name)), + Dir = rebar_utils:to_list(filename:join([DepsDir, Name, SubDir])), {ok, AppInfo0} = case rebar_app_info:discover(Dir) of {ok, App} -> @@ -267,12 +269,24 @@ dep_to_app(Parent, DepsDir, Name, Vsn, Source, IsLock, State) -> not_found -> rebar_app_info:new(Parent, Name, Vsn, Dir, []) end, - rebar_app_info:source(AppInfo0, Source) + rebar_app_info:source(AppInfo0, Source0) end, Overrides = rebar_app_info:get(AppInfo, overrides, []) ++ rebar_state:get(State, overrides, []), AppInfo2 = rebar_app_info:set(AppInfo, overrides, Overrides), AppInfo5 = rebar_app_info:profiles(AppInfo2, [default]), - rebar_app_info:is_lock(AppInfo5, IsLock). + AppInfo6 = rebar_app_info:fetch_dir(AppInfo5, FetchDir), + rebar_app_info:is_lock(AppInfo6, IsLock). + +%% git has to have special treatment. +%% unlike a resource that simply copies files the git resource needs to be able +%% to access the .git directory and run `git' commands to check against the lock +%% and optionally get the version. Because of this we must keep the clone as a +%% usable git repo clone and using a subdir can not be copied out of it but must +%% have the app info set its directory to be a sub directory of the repo. +subdir({git_subdir, _Repo, _Ref, Dir}) -> + Dir; +subdir(_) -> + "". %% @doc Takes a given application app_info record along with the project. %% If the app is a package, resolve and expand the package definition. diff --git a/src/rebar_fetch.erl b/src/rebar_fetch.erl index 9c76e0ea..0da2a1dd 100644 --- a/src/rebar_fetch.erl +++ b/src/rebar_fetch.erl @@ -54,9 +54,10 @@ download_source_(AppInfo, State) -> ok -> ec_file:mkdir_p(AppDir1), code:del_path(filename:absname(filename:join(AppDir1, "ebin"))), - ok = rebar_file_utils:rm_rf(filename:absname(AppDir1)), - ?DEBUG("Moving checkout ~p to ~p", [TmpDir, filename:absname(AppDir1)]), - rebar_file_utils:mv(TmpDir, filename:absname(AppDir1)); + FetchDir = rebar_app_info:fetch_dir(AppInfo), + ok = rebar_file_utils:rm_rf(filename:absname(FetchDir)), + ?DEBUG("Moving checkout ~p to ~p", [TmpDir, filename:absname(FetchDir)]), + rebar_file_utils:mv(TmpDir, filename:absname(FetchDir)); Error -> Error end. diff --git a/src/rebar_git_resource.erl b/src/rebar_git_resource.erl index e21d9e89..b81d50b3 100644 --- a/src/rebar_git_resource.erl +++ b/src/rebar_git_resource.erl @@ -10,9 +10,15 @@ needs_update/2, make_vsn/2]). +%% for use by rebar_git_sparse_resource +-export([lock_/2, + download_/3, + needs_update_/2, + make_vsn_/1, + git_vsn/0]). + %% For backward compatibilty --export ([ download/3 - ]). +-export ([download/3]). -include("rebar.hrl"). @@ -28,29 +34,31 @@ lock(AppInfo, _) -> check_type_support(), lock_(rebar_app_info:dir(AppInfo), rebar_app_info:source(AppInfo)). -lock_(AppDir, {git, Url, _}) -> - lock_(AppDir, {git, Url}); lock_(AppDir, {git, Url}) -> + lock_(AppDir, {git, Url, [], []}); +lock_(AppDir, {git, Url, Ref}) -> + lock_(AppDir, {git, Url, Ref, []}); +lock_(AppDir, {git, Url, _, Opts}) -> AbortMsg = lists:flatten(io_lib:format("Locking of git dependency failed in ~ts", [AppDir])), Dir = rebar_utils:escape_double_quotes(AppDir), {ok, VsnString} = case os:type() of {win32, _} -> - rebar_utils:sh("git --git-dir=\"" ++ Dir ++ "/.git\" " + rebar_utils:sh("git -C \"" ++ Dir ++ "\" " "--work-tree=\"" ++ Dir ++ "\" rev-parse --verify HEAD", [{use_stdout, false}, {debug_abort_on_error, AbortMsg}]); _ -> - rebar_utils:sh("git --git-dir='" ++ Dir ++ "/.git' rev-parse --verify HEAD", + rebar_utils:sh("git -C '" ++ Dir ++ "' rev-parse --verify HEAD", [{use_stdout, false}, {debug_abort_on_error, AbortMsg}]) end, Ref = rebar_string:trim(VsnString, both, "\n"), - {git, Url, {ref, Ref}}. + {git, Url, {ref, Ref}, Opts}. %% Return true if either the git url or tag/branch/ref is not the same as the currently %% checked out git repo for the dep needs_update(AppInfo, _) -> check_type_support(), - needs_update_(rebar_app_info:dir(AppInfo), rebar_app_info:source(AppInfo)). + needs_update_(rebar_app_info:dir(AppInfo), rm_source_opts(rebar_app_info:source(AppInfo))). needs_update_(Dir, {git, Url, {tag, Tag}}) -> {ok, Current} = rebar_utils:sh(?FMT("git describe --tags --exact-match", []), @@ -120,6 +128,8 @@ parse_git_url(not_scp, Url) -> download(TmpDir, AppInfo, State, _) -> check_type_support(), case download_(TmpDir, rebar_app_info:source(AppInfo), State) of + ok -> + ok; {ok, _} -> ok; {error, Reason} -> @@ -128,29 +138,37 @@ download(TmpDir, AppInfo, State, _) -> {error, Error} end. +rm_source_opts({Type, Url, Ref}) -> + {Type, Url, Ref}; +rm_source_opts({Type, Url, Ref, _Opts}) -> + {Type, Url, Ref}. + %% For backward compatibilty download(Dir, AppInfo, State) -> download_(Dir, AppInfo, State). download_(Dir, {git, Url}, State) -> ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []), - download_(Dir, {git, Url, {branch, "master"}}, State); + download_(Dir, {git, Url, {branch, "master"}, []}, State); download_(Dir, {git, Url, ""}, State) -> ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []), - download_(Dir, {git, Url, {branch, "master"}}, State); -download_(Dir, {git, Url, {branch, Branch}}, _State) -> + download_(Dir, {git, Url, {branch, "master"}, []}, State); +download_(Dir, {git, Url, Checkout}, _State) -> + %% add an empty opts element to the tupel if there is none + download_(Dir, {git, Url, Checkout, []}, _State); +download_(Dir, {git, Url, {branch, Branch}, _Opts}, _State) -> ok = filelib:ensure_dir(Dir), maybe_warn_local_url(Url), git_clone(branch, git_vsn(), Url, Dir, Branch); -download_(Dir, {git, Url, {tag, Tag}}, _State) -> +download_(Dir, {git, Url, {tag, Tag}, _Opts}, _State) -> ok = filelib:ensure_dir(Dir), maybe_warn_local_url(Url), git_clone(tag, git_vsn(), Url, Dir, Tag); -download_(Dir, {git, Url, {ref, Ref}}, _State) -> +download_(Dir, {git, Url, {ref, Ref}, _Opts}, _State) -> ok = filelib:ensure_dir(Dir), maybe_warn_local_url(Url), git_clone(ref, git_vsn(), Url, Dir, Ref); -download_(Dir, {git, Url, Rev}, _State) -> +download_(Dir, {git, Url, Rev, _Opts}, _State) -> ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []), ok = filelib:ensure_dir(Dir), maybe_warn_local_url(Url), @@ -167,57 +185,60 @@ maybe_warn_local_url(Url) -> end. %% Use different git clone commands depending on git --version -git_clone(branch,Vsn,Url,Dir,Branch) when Vsn >= {1,7,10}; Vsn =:= undefined -> +git_clone(branch, GitVsn, Url, Dir, Branch) when GitVsn >= {1,7,10}; GitVsn =:= undefined -> rebar_utils:sh(?FMT("git clone ~ts ~ts ~ts -b ~ts --single-branch", [git_clone_options(), rebar_utils:escape_chars(Url), rebar_utils:escape_chars(filename:basename(Dir)), rebar_utils:escape_chars(Branch)]), - [{cd, filename:dirname(Dir)}]); -git_clone(branch,_Vsn,Url,Dir,Branch) -> + [{cd, filename:dirname(Dir)}]), + ok; +git_clone(branch, _GitVsn, Url, Dir, Branch) -> rebar_utils:sh(?FMT("git clone ~ts ~ts ~ts -b ~ts", [git_clone_options(), rebar_utils:escape_chars(Url), rebar_utils:escape_chars(filename:basename(Dir)), rebar_utils:escape_chars(Branch)]), - [{cd, filename:dirname(Dir)}]); -git_clone(tag,Vsn,Url,Dir,Tag) when Vsn >= {1,7,10}; Vsn =:= undefined -> + [{cd, filename:dirname(Dir)}]), + ok; +git_clone(tag, GitVsn, Url, Dir, Tag) when GitVsn >= {1,7,10}; GitVsn =:= undefined -> rebar_utils:sh(?FMT("git clone ~ts ~ts ~ts -b ~ts --single-branch", [git_clone_options(), rebar_utils:escape_chars(Url), rebar_utils:escape_chars(filename:basename(Dir)), rebar_utils:escape_chars(Tag)]), - [{cd, filename:dirname(Dir)}]); -git_clone(tag,_Vsn,Url,Dir,Tag) -> + [{cd, filename:dirname(Dir)}]), + ok; +git_clone(tag, _GitVsn, Url, Dir,Tag) -> rebar_utils:sh(?FMT("git clone ~ts ~ts ~ts -b ~ts", [git_clone_options(), rebar_utils:escape_chars(Url), rebar_utils:escape_chars(filename:basename(Dir)), rebar_utils:escape_chars(Tag)]), - [{cd, filename:dirname(Dir)}]); -git_clone(ref,_Vsn,Url,Dir,Ref) -> + [{cd, filename:dirname(Dir)}]), + ok; +git_clone(ref, _GitVsn, Url, Dir, Ref) -> rebar_utils:sh(?FMT("git clone ~ts -n ~ts ~ts", [git_clone_options(), rebar_utils:escape_chars(Url), rebar_utils:escape_chars(filename:basename(Dir))]), [{cd, filename:dirname(Dir)}]), - rebar_utils:sh(?FMT("git checkout -q ~ts", [Ref]), [{cd, Dir}]); -git_clone(rev,_Vsn,Url,Dir,Rev) -> + rebar_utils:sh(?FMT("git checkout -q ~ts", [rebar_utils:escape_chars(Ref)]), [{cd, Dir}]); +git_clone(rev, _GitVsn, Url, Dir, Rev) -> rebar_utils:sh(?FMT("git clone ~ts -n ~ts ~ts", [git_clone_options(), rebar_utils:escape_chars(Url), rebar_utils:escape_chars(filename:basename(Dir))]), [{cd, filename:dirname(Dir)}]), - rebar_utils:sh(?FMT("git checkout -q ~ts", [rebar_utils:escape_chars(Rev)]), - [{cd, Dir}]). + rebar_utils:sh(?FMT("git checkout -q ~ts", [rebar_utils:escape_chars(Rev)]), [{cd, Dir}]). git_vsn() -> case application:get_env(rebar, git_vsn) of - {ok, Vsn} -> Vsn; + {ok, GitVsn} -> GitVsn; undefined -> - Vsn = git_vsn_fetch(), - application:set_env(rebar, git_vsn, Vsn), - Vsn + GitVsn = git_vsn_fetch(), + application:set_env(rebar, git_vsn, GitVsn), + GitVsn end. git_vsn_fetch() -> diff --git a/src/rebar_git_subdir_resource.erl b/src/rebar_git_subdir_resource.erl new file mode 100644 index 00000000..65c588d8 --- /dev/null +++ b/src/rebar_git_subdir_resource.erl @@ -0,0 +1,77 @@ +%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ts=4 sw=4 et +-module(rebar_git_subdir_resource). + +-behaviour(rebar_resource_v2). + +-export([init/2, + lock/2, + download/4, + needs_update/2, + make_vsn/2]). + +-include("rebar.hrl"). + +%% Regex used for parsing scp style remote url +-define(SCP_PATTERN, "\\A(?[^@]+)@(?[^:]+):(?.+)\\z"). + +-spec init(atom(), rebar_state:t()) -> {ok, rebar_resource_v2:resource()}. +init(Type, _State) -> + Resource = rebar_resource_v2:new(Type, ?MODULE, #{}), + {ok, Resource}. + +lock(AppInfo, _) -> + {git_subdir, Url, Checkout, Dir} = rebar_app_info:source(AppInfo), + {git, Url1, {ref, Ref}, _Opts} = + rebar_git_resource:lock_(rebar_app_info:dir(AppInfo), {git, Url, Checkout}), + {git_subdir, Url1, {ref, Ref}, Dir}. + +download(TmpDir, AppInfo, State, _) -> + {git_subdir, Url, Checkout, SparseDir} = rebar_app_info:source(AppInfo), + case rebar_git_resource:download_(TmpDir, {git, Url, Checkout}, State) of + ok -> + sparse_checkout(rebar_git_resource:git_vsn(), TmpDir, to_ref(Checkout), SparseDir); + {ok, _} -> + sparse_checkout(rebar_git_resource:git_vsn(), TmpDir, to_ref(Checkout), SparseDir); + {error, Reason} -> + {error, Reason}; + Error -> + {error, Error} + end. + +%% Return true if either the git url or tag/branch/ref is not the same as the currently +%% checked out git repo for the dep +needs_update(AppInfo, _) -> + {git_subdir, Url, Ref, _Dir} = rebar_app_info:source(AppInfo), + rebar_git_resource:needs_update_(rebar_app_info:dir(AppInfo), {git, Url, Ref}). + +make_vsn(AppInfo, _) -> + Dir = rebar_app_info:dir(AppInfo), + rebar_git_resource:make_vsn_(Dir). + +%% + +to_ref({branch, Branch}) -> + Branch; +to_ref({tag, Tag}) -> + Tag; +to_ref({ref, Ref}) -> + Ref; +to_ref(Rev) -> + Rev. + +sparse_checkout(GitVsn, Dir, Ref, SparseDir) when GitVsn >= {1,7,4}; + GitVsn =:= undefined -> + ?DEBUG("doing sparse checkout in ~s of dir ~s", [Dir, SparseDir]), + + rebar_utils:sh(?FMT("git --git-dir=.git config core.sparsecheckout true", []), + [{cd, Dir}]), + filelib:ensure_dir(filename:join(Dir, ".git/info/sparse-checkout")), + file:write_file(filename:join(Dir, ".git/info/sparse-checkout"), SparseDir), + rebar_utils:sh(?FMT("git checkout -q ~ts", [rebar_utils:escape_chars(Ref)]), [{cd, Dir}]), + ok; +sparse_checkout(_, _, _, _) -> + %% sparse checkout not supported but we can still use the subdirectory + %% so no need to fail, just don't do the sparse checkout + ?DEBUG("too old a git version to do a sparse checkout for a subdir dep", []), + ok.