git_subdir resource for dependency that is in a subdirectory of repo

Because the resource doesn't absolutely require using a sparse
checkout, it only matters that rebar3 knows the correct path of the
application, the resource is named git_subdir.

It has to be git specific because the reason this subdir support is
special is the need to keep the code checkout as a usable git clone
to run commands against. This means the resource can't just copy
the content of the subdirectory to the application out directory.

rebar.lock

@@ -10,7 +10,8 @@
  {<<"providers">>,{pkg,<<"providers">>,<<"1.8.1">>},0},
  {<<"relx">>,
   {git,"https://github.com/erlware/relx.git",
-       {ref,"ac5bb88330e23b29ae0449214af5f62bd16ecdce"}},
+       {ref,"ac5bb88330e23b29ae0449214af5f62bd16ecdce"},
+       []},
   0},
  {<<"ssl_verify_fun">>,{pkg,<<"ssl_verify_fun">>,<<"1.1.5">>},0}]}.
 [

src/rebar.app.src

@@ -36,6 +36,7 @@
         {log_level, warn},
 
         {resources, [{git, rebar_git_resource},
+                     {git_subdir, rebar_git_subdir_resource},
                      {pkg, rebar_pkg_resource},
                      {hg, rebar_hg_resource}]},
 

src/rebar_app_info.erl

@@ -36,6 +36,8 @@
          deps/2,
          dep_level/1,
          dep_level/2,
+         fetch_dir/1,
+         fetch_dir/2,
          dir/1,
          dir/2,
          out_dir/1,
@@ -79,29 +81,30 @@
 
 -type project_type() :: rebar3 | mix | undefined.
 
--record(app_info_t, {name                     :: binary() | undefined,
-                     app_file_src             :: file:filename_all() | undefined,
-                     app_file_src_script      :: file:filename_all() | undefined,
-                     app_file                 :: file:filename_all() | undefined,
-                     original_vsn             :: binary() | undefined,
-                     parent=root              :: binary() | root,
-                     app_details=[]           :: list(),
-                     applications=[]          :: list(),
+-record(app_info_t, {name               :: binary() | undefined,
+                     app_file_src       :: file:filename_all() | undefined,
+                     
app_file_src_script:: file:filename_all() | undefined,
+                     app_file           :: file:filename_all() | undefined,
+                     original_vsn       :: binary() | undefined,
+                     parent=root        :: binary() | root,
+                     app_details=[]     :: list(),
+                     applications=[]    :: list(),
                      included_applications=[] :: [atom()],
-                     deps=[]                  :: list(),
-                     profiles=[default]       :: [atom()],
-                     default=dict:new()       :: rebar_dict(),
-                     opts=dict:new()          :: rebar_dict(),
-                     dep_level=0              :: integer(),
-                     dir                      :: file:name(),
-                     out_dir                  :: file:name(),
-                     ebin_dir                 :: file:name(),
-                     source                   :: string() | tuple() | checkout | undefined,
-                     is_lock=false            :: boolean(),
-                     is_checkout=false        :: boolean(),
-                     valid                    :: boolean() | undefined,
-                     project_type             :: project_type(),
-                     is_available=false       :: boolean()}).
+                     deps=[]            :: list(),
+                     profiles=[default] :: [atom()],
+                     default=dict:new() :: rebar_dict(),
+                     opts=dict:new()    :: rebar_dict(),
+                     dep_level=0        :: integer(),
+                     fetch_dir          :: file:name(),
+                     dir                :: file:name(),
+                     out_dir            :: file:name(),
+                     ebin_dir           :: file:name(),
+                     source             :: string() | tuple() | checkout | undefined,
+                     is_lock=false      :: boolean(),
+                     is_checkout=false  :: boolean(),
+                     valid              :: boolean() | undefined,
+                     project_type       :: project_type(),
+                     is_available=false :: boolean()}).
 
 %%============================================================================
 %% types
@@ -136,6 +139,7 @@ new(AppName, Vsn) ->
 new(AppName, Vsn, Dir) ->
     {ok, #app_info_t{name=rebar_utils:to_binary(AppName),
                      original_vsn=Vsn,
+                     fetch_dir=rebar_utils:to_list(Dir),
                      dir=rebar_utils:to_list(Dir),
                      out_dir=rebar_utils:to_list(Dir),
                      ebin_dir=filename:join(rebar_utils:to_list(Dir), "ebin")}}.
@@ -146,6 +150,7 @@ new(AppName, Vsn, Dir) ->
 new(AppName, Vsn, Dir, Deps) ->
     {ok, #app_info_t{name=rebar_utils:to_binary(AppName),
                      original_vsn=Vsn,
+                     fetch_dir=rebar_utils:to_list(Dir),
                      dir=rebar_utils:to_list(Dir),
                      out_dir=rebar_utils:to_list(Dir),
                      ebin_dir=filename:join(rebar_utils:to_list(Dir), "ebin"),
@@ -158,6 +163,7 @@ new(Parent, AppName, Vsn, Dir, Deps) ->
     {ok, #app_info_t{name=rebar_utils:to_binary(AppName),
                      parent=Parent,
                      original_vsn=Vsn,
+                     fetch_dir=rebar_utils:to_list(Dir),
                      dir=rebar_utils:to_list(Dir),
                      out_dir=rebar_utils:to_list(Dir),
                      ebin_dir=filename:join(rebar_utils:to_list(Dir), "ebin"),
@@ -469,6 +475,16 @@ dep_level(#app_info_t{dep_level=Level}) ->
 dep_level(AppInfo=#app_info_t{}, Level) ->
     AppInfo#app_info_t{dep_level=Level}.
 
+%% @doc returns the directory to fetch the dep source to
+-spec fetch_dir(t()) -> file:name().
+fetch_dir(#app_info_t{fetch_dir=FetchDir}) ->
+    FetchDir.
+
+%% @doc returns the directory to fetch the dep source to
+-spec fetch_dir(t(), file:name()) -> file:name().
+fetch_dir(AppInfo=#app_info_t{}, FetchDir) ->
+    AppInfo#app_info_t{fetch_dir=FetchDir}.
+
 %% @doc returns the directory that contains the app.
 -spec dir(t()) -> file:name().
 dir(#app_info_t{dir=Dir}) ->

src/rebar_app_utils.erl

@@ -251,13 +251,15 @@ parse_dep(_, Dep, _, _, _) ->
       Source :: tuple(),
       IsLock :: boolean(),
       State :: rebar_state:t().
-dep_to_app(Parent, DepsDir, Name, Vsn, Source, IsLock, State) ->
+dep_to_app(Parent, DepsDir, Name, Vsn, Source0, IsLock, State) ->
+    SubDir = subdir(Source0),
+    FetchDir = rebar_utils:to_list(filename:join([DepsDir, Name])),
     CheckoutsDir = rebar_utils:to_list(rebar_dir:checkouts_dir(State, Name)),
     AppInfo = case rebar_app_info:discover(CheckoutsDir) of
                   {ok, App} ->
                       rebar_app_info:source(rebar_app_info:is_checkout(App, true), checkout);
                   not_found ->
-                      Dir = rebar_utils:to_list(filename:join(DepsDir, Name)),
+                      Dir = rebar_utils:to_list(filename:join([DepsDir, Name, SubDir])),
                       {ok, AppInfo0} =
                           case rebar_app_info:discover(Dir) of
                               {ok, App} ->
@@ -267,12 +269,24 @@ dep_to_app(Parent, DepsDir, Name, Vsn, Source, IsLock, State) ->
                               not_found ->
                                   rebar_app_info:new(Parent, Name, Vsn, Dir, [])
                           end,
-                      rebar_app_info:source(AppInfo0, Source)
+                      rebar_app_info:source(AppInfo0, Source0)
               end,
     Overrides = rebar_app_info:get(AppInfo, overrides, []) ++ rebar_state:get(State, overrides, []),
     AppInfo2 = rebar_app_info:set(AppInfo, overrides, Overrides),
     AppInfo5 = rebar_app_info:profiles(AppInfo2, [default]),
-    rebar_app_info:is_lock(AppInfo5, IsLock).
+    AppInfo6 = rebar_app_info:fetch_dir(AppInfo5, FetchDir),
+    rebar_app_info:is_lock(AppInfo6, IsLock).
+
+%% git has to have special treatment.
+%% unlike a resource that simply copies files the git resource needs to be able
+%% to access the .git directory and run `git' commands to check against the lock
+%% and optionally get the version. Because of this we must keep the clone as a
+%% usable git repo clone and using a subdir can not be copied out of it but must
+%% have the app info set its directory to be a sub directory of the repo.
+subdir({git_subdir, _Repo, _Ref, Dir}) ->
+    Dir;
+subdir(_) ->
+    "".
 
 %% @doc Takes a given application app_info record along with the project.
 %% If the app is a package, resolve and expand the package definition.

src/rebar_fetch.erl

@@ -54,9 +54,10 @@ download_source_(AppInfo, State) ->
         ok ->
             ec_file:mkdir_p(AppDir1),
             code:del_path(filename:absname(filename:join(AppDir1, "ebin"))),
-            ok = rebar_file_utils:rm_rf(filename:absname(AppDir1)),
-            ?DEBUG("Moving checkout ~p to ~p", [TmpDir, filename:absname(AppDir1)]),
-            rebar_file_utils:mv(TmpDir, filename:absname(AppDir1));
+            FetchDir = rebar_app_info:fetch_dir(AppInfo),
+            ok = rebar_file_utils:rm_rf(filename:absname(FetchDir)),
+            ?DEBUG("Moving checkout ~p to ~p", [TmpDir, filename:absname(FetchDir)]),
+            rebar_file_utils:mv(TmpDir, filename:absname(FetchDir));
         Error ->
             Error
     end.

src/rebar_git_resource.erl

@@ -10,9 +10,15 @@
          needs_update/2,
          make_vsn/2]).
 
+%% for use by rebar_git_sparse_resource
+-export([lock_/2,
+         download_/3,
+         needs_update_/2,
+         make_vsn_/1,
+         git_vsn/0]).
+
 %% For backward compatibilty
--export ([ download/3
-         ]).
+-export ([download/3]).
 
 -include("rebar.hrl").
 
@@ -28,29 +34,31 @@ lock(AppInfo, _) ->
     check_type_support(),
     lock_(rebar_app_info:dir(AppInfo), rebar_app_info:source(AppInfo)).
 
-lock_(AppDir, {git, Url, _}) ->
-    lock_(AppDir, {git, Url});
 lock_(AppDir, {git, Url}) ->
+    lock_(AppDir, {git, Url, [], []});
+lock_(AppDir, {git, Url, Ref}) ->
+    lock_(AppDir, {git, Url, Ref, []});
+lock_(AppDir, {git, Url, _, Opts}) ->
     AbortMsg = lists:flatten(io_lib:format("Locking of git dependency failed in ~ts", [AppDir])),
     Dir = rebar_utils:escape_double_quotes(AppDir),
     {ok, VsnString} =
         case os:type() of
             {win32, _} ->
-                rebar_utils:sh("git --git-dir=\"" ++ Dir ++ "/.git\" "
+                rebar_utils:sh("git -C \"" ++ Dir ++ "\" "
                                "--work-tree=\"" ++ Dir ++ "\" rev-parse --verify HEAD",
                     [{use_stdout, false}, {debug_abort_on_error, AbortMsg}]);
             _ ->
-                rebar_utils:sh("git --git-dir='" ++ Dir ++ "/.git' rev-parse --verify HEAD",
+                rebar_utils:sh("git -C '" ++ Dir ++ "' rev-parse --verify HEAD",
                     [{use_stdout, false}, {debug_abort_on_error, AbortMsg}])
         end,
     Ref = rebar_string:trim(VsnString, both, "\n"),
-    {git, Url, {ref, Ref}}.
+    {git, Url, {ref, Ref}, Opts}.
 
 %% Return true if either the git url or tag/branch/ref is not the same as the currently
 %% checked out git repo for the dep
 needs_update(AppInfo, _) ->
     check_type_support(),
-    needs_update_(rebar_app_info:dir(AppInfo), rebar_app_info:source(AppInfo)).
+    needs_update_(rebar_app_info:dir(AppInfo), rm_source_opts(rebar_app_info:source(AppInfo))).
 
 needs_update_(Dir, {git, Url, {tag, Tag}}) ->
     {ok, Current} = rebar_utils:sh(?FMT("git describe --tags --exact-match", []),
@@ -120,6 +128,8 @@ parse_git_url(not_scp, Url) ->
 download(TmpDir, AppInfo, State, _) ->
     check_type_support(),
     case download_(TmpDir, rebar_app_info:source(AppInfo), State) of
+        ok ->
+            ok;
         {ok, _} ->
             ok;
         {error, Reason} ->
@@ -128,29 +138,37 @@ download(TmpDir, AppInfo, State, _) ->
             {error, Error}
     end.
 
+rm_source_opts({Type, Url, Ref}) ->
+    {Type, Url, Ref};
+rm_source_opts({Type, Url, Ref, _Opts}) ->
+    {Type, Url, Ref}.
+
 %% For backward compatibilty
 download(Dir, AppInfo, State) ->
     download_(Dir, AppInfo, State).
 
 download_(Dir, {git, Url}, State) ->
     ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []),
-    download_(Dir, {git, Url, {branch, "master"}}, State);
+    download_(Dir, {git, Url, {branch, "master"}, []}, State);
 download_(Dir, {git, Url, ""}, State) ->
     ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []),
-    download_(Dir, {git, Url, {branch, "master"}}, State);
-download_(Dir, {git, Url, {branch, Branch}}, _State) ->
+    download_(Dir, {git, Url, {branch, "master"}, []}, State);
+download_(Dir, {git, Url, Checkout}, _State) ->
+    %% add an empty opts element to the tupel if there is none
+    download_(Dir, {git, Url, Checkout, []}, _State);
+download_(Dir, {git, Url, {branch, Branch}, _Opts}, _State) ->
     ok = filelib:ensure_dir(Dir),
     maybe_warn_local_url(Url),
     git_clone(branch, git_vsn(), Url, Dir, Branch);
-download_(Dir, {git, Url, {tag, Tag}}, _State) ->
+download_(Dir, {git, Url, {tag, Tag}, _Opts}, _State) ->
     ok = filelib:ensure_dir(Dir),
     maybe_warn_local_url(Url),
     git_clone(tag, git_vsn(), Url, Dir, Tag);
-download_(Dir, {git, Url, {ref, Ref}}, _State) ->
+download_(Dir, {git, Url, {ref, Ref}, _Opts}, _State) ->
     ok = filelib:ensure_dir(Dir),
     maybe_warn_local_url(Url),
     git_clone(ref, git_vsn(), Url, Dir, Ref);
-download_(Dir, {git, Url, Rev}, _State) ->
+download_(Dir, {git, Url, Rev, _Opts}, _State) ->
     ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []),
     ok = filelib:ensure_dir(Dir),
     maybe_warn_local_url(Url),
@@ -167,57 +185,60 @@ maybe_warn_local_url(Url) ->
     end.
 
 %% Use different git clone commands depending on git --version
-git_clone(branch,Vsn,Url,Dir,Branch) when Vsn >= {1,7,10}; Vsn =:= undefined ->
+git_clone(branch, GitVsn, Url, Dir, Branch) when GitVsn >= {1,7,10}; GitVsn =:= undefined ->
     rebar_utils:sh(?FMT("git clone ~ts ~ts ~ts -b ~ts --single-branch",
                         [git_clone_options(),
                          rebar_utils:escape_chars(Url),
                          rebar_utils:escape_chars(filename:basename(Dir)),
                          rebar_utils:escape_chars(Branch)]),
-                   [{cd, filename:dirname(Dir)}]);
-git_clone(branch,_Vsn,Url,Dir,Branch) ->
+                   [{cd, filename:dirname(Dir)}]),
+    ok;
+git_clone(branch, _GitVsn, Url, Dir, Branch) ->
     rebar_utils:sh(?FMT("git clone ~ts ~ts ~ts -b ~ts",
                         [git_clone_options(),
                          rebar_utils:escape_chars(Url),
                          rebar_utils:escape_chars(filename:basename(Dir)),
                          rebar_utils:escape_chars(Branch)]),
-                   [{cd, filename:dirname(Dir)}]);
-git_clone(tag,Vsn,Url,Dir,Tag) when Vsn >= {1,7,10}; Vsn =:= undefined ->
+                   [{cd, filename:dirname(Dir)}]),
+    ok;
+git_clone(tag, GitVsn, Url, Dir, Tag) when GitVsn >= {1,7,10}; GitVsn =:= undefined ->
     rebar_utils:sh(?FMT("git clone ~ts ~ts ~ts -b ~ts --single-branch",
                         [git_clone_options(),
                          rebar_utils:escape_chars(Url),
                          rebar_utils:escape_chars(filename:basename(Dir)),
                          rebar_utils:escape_chars(Tag)]),
-                   [{cd, filename:dirname(Dir)}]);
-git_clone(tag,_Vsn,Url,Dir,Tag) ->
+                   [{cd, filename:dirname(Dir)}]),
+    ok;
+git_clone(tag, _GitVsn, Url, Dir,Tag) ->
     rebar_utils:sh(?FMT("git clone ~ts ~ts ~ts -b ~ts",
                         [git_clone_options(),
                          rebar_utils:escape_chars(Url),
                          rebar_utils:escape_chars(filename:basename(Dir)),
                          rebar_utils:escape_chars(Tag)]),
-                   [{cd, filename:dirname(Dir)}]);
-git_clone(ref,_Vsn,Url,Dir,Ref) ->
+                   [{cd, filename:dirname(Dir)}]),
+    ok;
+git_clone(ref, _GitVsn, Url, Dir, Ref) ->
     rebar_utils:sh(?FMT("git clone ~ts -n ~ts ~ts",
                         [git_clone_options(),
                          rebar_utils:escape_chars(Url),
                          rebar_utils:escape_chars(filename:basename(Dir))]),
                    [{cd, filename:dirname(Dir)}]),
-    rebar_utils:sh(?FMT("git checkout -q ~ts", [Ref]), [{cd, Dir}]);
-git_clone(rev,_Vsn,Url,Dir,Rev) ->
+    rebar_utils:sh(?FMT("git checkout -q ~ts", [rebar_utils:escape_chars(Ref)]), [{cd, Dir}]);
+git_clone(rev, _GitVsn, Url, Dir, Rev) ->
     rebar_utils:sh(?FMT("git clone ~ts -n ~ts ~ts",
                         [git_clone_options(),
                          rebar_utils:escape_chars(Url),
                          rebar_utils:escape_chars(filename:basename(Dir))]),
                    [{cd, filename:dirname(Dir)}]),
-    rebar_utils:sh(?FMT("git checkout -q ~ts", [rebar_utils:escape_chars(Rev)]),
-                   [{cd, Dir}]).
+    rebar_utils:sh(?FMT("git checkout -q ~ts", [rebar_utils:escape_chars(Rev)]), [{cd, Dir}]).
 
 git_vsn() ->
     case application:get_env(rebar, git_vsn) of
-        {ok, Vsn} -> Vsn;
+        {ok, GitVsn} -> GitVsn;
         undefined ->
-            Vsn = git_vsn_fetch(),
-            application:set_env(rebar, git_vsn, Vsn),
-            Vsn
+            GitVsn = git_vsn_fetch(),
+            application:set_env(rebar, git_vsn, GitVsn),
+            GitVsn
     end.
 
 git_vsn_fetch() ->

src/rebar_git_subdir_resource.erl

@@ -0,0 +1,77 @@
+%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*-
+%% ex: ts=4 sw=4 et
+-module(rebar_git_subdir_resource).
+
+-behaviour(rebar_resource_v2).
+
+-export([init/2,
+         lock/2,
+         download/4,
+         needs_update/2,
+         make_vsn/2]).
+
+-include("rebar.hrl").
+
+%% Regex used for parsing scp style remote url
+-define(SCP_PATTERN, "\\A(?<username>[^@]+)@(?<host>[^:]+):(?<path>.+)\\z").
+
+-spec init(atom(), rebar_state:t()) -> {ok, rebar_resource_v2:resource()}.
+init(Type, _State) ->
+    Resource = rebar_resource_v2:new(Type, ?MODULE, #{}),
+    {ok, Resource}.
+
+lock(AppInfo, _) ->
+    {git_subdir, Url, Checkout, Dir} = rebar_app_info:source(AppInfo),
+    {git, Url1, {ref, Ref}, _Opts} =
+        rebar_git_resource:lock_(rebar_app_info:dir(AppInfo), {git, Url, Checkout}),
+    {git_subdir, Url1, {ref, Ref}, Dir}.
+
+download(TmpDir, AppInfo, State, _) ->
+    {git_subdir, Url, Checkout, SparseDir} = rebar_app_info:source(AppInfo),
+    case rebar_git_resource:download_(TmpDir, {git, Url, Checkout}, State) of
+        ok ->
+            sparse_checkout(rebar_git_resource:git_vsn(), TmpDir, to_ref(Checkout), SparseDir);
+        {ok, _} ->
+            sparse_checkout(rebar_git_resource:git_vsn(), TmpDir, to_ref(Checkout), SparseDir);
+        {error, Reason} ->
+            {error, Reason};
+        Error ->
+            {error, Error}
+    end.
+
+%% Return true if either the git url or tag/branch/ref is not the same as the currently
+%% checked out git repo for the dep
+needs_update(AppInfo, _) ->
+    {git_subdir, Url, Ref, _Dir} = rebar_app_info:source(AppInfo),
+    rebar_git_resource:needs_update_(rebar_app_info:dir(AppInfo), {git, Url, Ref}).
+
+make_vsn(AppInfo, _) ->
+    Dir = rebar_app_info:dir(AppInfo),
+    rebar_git_resource:make_vsn_(Dir).
+
+%%
+
+to_ref({branch, Branch}) ->
+    Branch;
+to_ref({tag, Tag}) ->
+    Tag;
+to_ref({ref, Ref}) ->
+    Ref;
+to_ref(Rev) ->
+    Rev.
+
+sparse_checkout(GitVsn, Dir, Ref, SparseDir) when GitVsn >= {1,7,4};
+                                                  GitVsn =:= undefined  ->
+    ?DEBUG("doing sparse checkout in ~s of dir ~s", [Dir, SparseDir]),
+
+    rebar_utils:sh(?FMT("git --git-dir=.git config core.sparsecheckout true", []),
+                   [{cd, Dir}]),
+    filelib:ensure_dir(filename:join(Dir, ".git/info/sparse-checkout")),
+    file:write_file(filename:join(Dir, ".git/info/sparse-checkout"), SparseDir),
+    rebar_utils:sh(?FMT("git checkout -q ~ts", [rebar_utils:escape_chars(Ref)]), [{cd, Dir}]),
+    ok;
+sparse_checkout(_, _, _, _) ->
+    %% sparse checkout not supported but we can still use the subdirectory
+    %% so no need to fail, just don't do the sparse checkout
+    ?DEBUG("too old a git version to do a sparse checkout for a subdir dep", []),
+    ok.