Merge pull request #2239 from tsloughter/git-sparse

sparse-git checkout dependencies

rebar.lock

@@ -10,7 +10,8 @@
  {<<"providers">>,{pkg,<<"providers">>,<<"1.8.1">>},0},
  {<<"relx">>,
   {git,"https://github.com/erlware/relx.git",
-       {ref,"ac5bb88330e23b29ae0449214af5f62bd16ecdce"}},
+       {ref,"ac5bb88330e23b29ae0449214af5f62bd16ecdce"},
+       []},
   0},
  {<<"ssl_verify_fun">>,{pkg,<<"ssl_verify_fun">>,<<"1.1.5">>},0}]}.
 [

src/rebar.app.src

@@ -36,6 +36,7 @@
         {log_level, warn},
 
         {resources, [{git, rebar_git_resource},
+                     {git_subdir, rebar_git_subdir_resource},
                      {pkg, rebar_pkg_resource},
                      {hg, rebar_hg_resource}]},
 

src/rebar_app_discover.erl

@@ -216,8 +216,13 @@ reset_hooks(Opts, CurrentProfiles) ->
 -spec all_app_dirs([file:name()]) -> [{file:name(), [file:name()]}].
 all_app_dirs(LibDirs) ->
     lists:flatmap(fun(LibDir) ->
-                          {_, SrcDirs} = find_config_src(LibDir, ["src"]),
-                          app_dirs(LibDir, SrcDirs)
+                          case filelib:is_dir(LibDir) of
+                              true ->
+                                  {_, SrcDirs} = find_config_src(LibDir, ["src"]),
+                                  app_dirs(LibDir, SrcDirs);
+                              false ->
+                                  []
+                          end
                   end, LibDirs).
 
 %% @private find the directories for all apps based on their source dirs

src/rebar_app_info.erl

@@ -36,6 +36,8 @@
          deps/2,
          dep_level/1,
          dep_level/2,
+         fetch_dir/1,
+         fetch_dir/2,
          dir/1,
          dir/2,
          out_dir/1,
@@ -79,29 +81,30 @@
 
 -type project_type() :: rebar3 | mix | undefined.
 
--record(app_info_t, {name                     :: binary() | undefined,
-                     app_file_src             :: file:filename_all() | undefined,
-                     app_file_src_script      :: file:filename_all() | undefined,
-                     app_file                 :: file:filename_all() | undefined,
-                     original_vsn             :: binary() | undefined,
-                     parent=root              :: binary() | root,
-                     app_details=[]           :: list(),
-                     applications=[]          :: list(),
+-record(app_info_t, {name               :: binary() | undefined,
+                     app_file_src       :: file:filename_all() | undefined,
+                     
app_file_src_script:: file:filename_all() | undefined,
+                     app_file           :: file:filename_all() | undefined,
+                     original_vsn       :: binary() | undefined,
+                     parent=root        :: binary() | root,
+                     app_details=[]     :: list(),
+                     applications=[]    :: list(),
                      included_applications=[] :: [atom()],
-                     deps=[]                  :: list(),
-                     profiles=[default]       :: [atom()],
-                     default=dict:new()       :: rebar_dict(),
-                     opts=dict:new()          :: rebar_dict(),
-                     dep_level=0              :: integer(),
-                     dir                      :: file:name(),
-                     out_dir                  :: file:name(),
-                     ebin_dir                 :: file:name(),
-                     source                   :: string() | tuple() | checkout | undefined,
-                     is_lock=false            :: boolean(),
-                     is_checkout=false        :: boolean(),
-                     valid                    :: boolean() | undefined,
-                     project_type             :: project_type(),
-                     is_available=false       :: boolean()}).
+                     deps=[]            :: list(),
+                     profiles=[default] :: [atom()],
+                     default=dict:new() :: rebar_dict(),
+                     opts=dict:new()    :: rebar_dict(),
+                     dep_level=0        :: integer(),
+                     fetch_dir          :: file:name(),
+                     dir                :: file:name(),
+                     out_dir            :: file:name(),
+                     ebin_dir           :: file:name(),
+                     source             :: string() | tuple() | checkout | undefined,
+                     is_lock=false      :: boolean(),
+                     is_checkout=false  :: boolean(),
+                     valid              :: boolean() | undefined,
+                     project_type       :: project_type(),
+                     is_available=false :: boolean()}).
 
 %%============================================================================
 %% types
@@ -136,6 +139,7 @@ new(AppName, Vsn) ->
 new(AppName, Vsn, Dir) ->
     {ok, #app_info_t{name=rebar_utils:to_binary(AppName),
                      original_vsn=Vsn,
+                     fetch_dir=rebar_utils:to_list(Dir),
                      dir=rebar_utils:to_list(Dir),
                      out_dir=rebar_utils:to_list(Dir),
                      ebin_dir=filename:join(rebar_utils:to_list(Dir), "ebin")}}.
@@ -146,6 +150,7 @@ new(AppName, Vsn, Dir) ->
 new(AppName, Vsn, Dir, Deps) ->
     {ok, #app_info_t{name=rebar_utils:to_binary(AppName),
                      original_vsn=Vsn,
+                     fetch_dir=rebar_utils:to_list(Dir),
                      dir=rebar_utils:to_list(Dir),
                      out_dir=rebar_utils:to_list(Dir),
                      ebin_dir=filename:join(rebar_utils:to_list(Dir), "ebin"),
@@ -158,6 +163,7 @@ new(Parent, AppName, Vsn, Dir, Deps) ->
     {ok, #app_info_t{name=rebar_utils:to_binary(AppName),
                      parent=Parent,
                      original_vsn=Vsn,
+                     fetch_dir=rebar_utils:to_list(Dir),
                      dir=rebar_utils:to_list(Dir),
                      out_dir=rebar_utils:to_list(Dir),
                      ebin_dir=filename:join(rebar_utils:to_list(Dir), "ebin"),
@@ -469,6 +475,16 @@ dep_level(#app_info_t{dep_level=Level}) ->
 dep_level(AppInfo=#app_info_t{}, Level) ->
     AppInfo#app_info_t{dep_level=Level}.
 
+%% @doc returns the directory to fetch the dep source to
+-spec fetch_dir(t()) -> file:name().
+fetch_dir(#app_info_t{fetch_dir=FetchDir}) ->
+    FetchDir.
+
+%% @doc returns the directory to fetch the dep source to
+-spec fetch_dir(t(), file:name()) -> file:name().
+fetch_dir(AppInfo=#app_info_t{}, FetchDir) ->
+    AppInfo#app_info_t{fetch_dir=FetchDir}.
+
 %% @doc returns the directory that contains the app.
 -spec dir(t()) -> file:name().
 dir(#app_info_t{dir=Dir}) ->

src/rebar_app_utils.erl

@@ -251,13 +251,15 @@ parse_dep(_, Dep, _, _, _) ->
       Source :: tuple(),
       IsLock :: boolean(),
       State :: rebar_state:t().
-dep_to_app(Parent, DepsDir, Name, Vsn, Source, IsLock, State) ->
+dep_to_app(Parent, DepsDir, Name, Vsn, Source0, IsLock, State) ->
+    SubDir = subdir(Source0),
+    FetchDir = rebar_utils:to_list(filename:join([DepsDir, Name])),
     CheckoutsDir = rebar_utils:to_list(rebar_dir:checkouts_dir(State, Name)),
     AppInfo = case rebar_app_info:discover(CheckoutsDir) of
                   {ok, App} ->
                       rebar_app_info:source(rebar_app_info:is_checkout(App, true), checkout);
                   not_found ->
-                      Dir = rebar_utils:to_list(filename:join(DepsDir, Name)),
+                      Dir = rebar_utils:to_list(filename:join([DepsDir, Name, SubDir])),
                       {ok, AppInfo0} =
                           case rebar_app_info:discover(Dir) of
                               {ok, App} ->
@@ -267,12 +269,24 @@ dep_to_app(Parent, DepsDir, Name, Vsn, Source, IsLock, State) ->
                               not_found ->
                                   rebar_app_info:new(Parent, Name, Vsn, Dir, [])
                           end,
-                      rebar_app_info:source(AppInfo0, Source)
+                      rebar_app_info:source(AppInfo0, Source0)
               end,
     Overrides = rebar_app_info:get(AppInfo, overrides, []) ++ rebar_state:get(State, overrides, []),
     AppInfo2 = rebar_app_info:set(AppInfo, overrides, Overrides),
     AppInfo5 = rebar_app_info:profiles(AppInfo2, [default]),
-    rebar_app_info:is_lock(AppInfo5, IsLock).
+    AppInfo6 = rebar_app_info:fetch_dir(AppInfo5, FetchDir),
+    rebar_app_info:is_lock(AppInfo6, IsLock).
+
+%% git has to have special treatment.
+%% unlike a resource that simply copies files the git resource needs to be able
+%% to access the .git directory and run `git' commands to check against the lock
+%% and optionally get the version. Because of this we must keep the clone as a
+%% usable git repo clone and using a subdir can not be copied out of it but must
+%% have the app info set its directory to be a sub directory of the repo.
+subdir({git_subdir, _Repo, _Ref, Dir}) ->
+    Dir;
+subdir(_) ->
+    "".
 
 %% @doc Takes a given application app_info record along with the project.
 %% If the app is a package, resolve and expand the package definition.

src/rebar_fetch.erl

@@ -54,9 +54,10 @@ download_source_(AppInfo, State) ->
         ok ->
             ec_file:mkdir_p(AppDir1),
             code:del_path(filename:absname(filename:join(AppDir1, "ebin"))),
-            ok = rebar_file_utils:rm_rf(filename:absname(AppDir1)),
-            ?DEBUG("Moving checkout ~p to ~p", [TmpDir, filename:absname(AppDir1)]),
-            rebar_file_utils:mv(TmpDir, filename:absname(AppDir1));
+            FetchDir = rebar_app_info:fetch_dir(AppInfo),
+            ok = rebar_file_utils:rm_rf(filename:absname(FetchDir)),
+            ?DEBUG("Moving checkout ~p to ~p", [TmpDir, filename:absname(FetchDir)]),
+            rebar_file_utils:mv(TmpDir, filename:absname(FetchDir));
         Error ->
             Error
     end.

src/rebar_git_resource.erl

@@ -10,9 +10,15 @@
          needs_update/2,
          make_vsn/2]).
 
+%% for use by rebar_git_sparse_resource
+-export([lock_/2,
+         download_/3,
+         needs_update_/2,
+         make_vsn_/1,
+         git_vsn/0]).
+
 %% For backward compatibilty
--export ([ download/3
-         ]).
+-export ([download/3]).
 
 -include("rebar.hrl").
 
@@ -28,29 +34,31 @@ lock(AppInfo, _) ->
     check_type_support(),
     lock_(rebar_app_info:dir(AppInfo), rebar_app_info:source(AppInfo)).
 
-lock_(AppDir, {git, Url, _}) ->
-    lock_(AppDir, {git, Url});
 lock_(AppDir, {git, Url}) ->
+    lock_(AppDir, {git, Url, [], []});
+lock_(AppDir, {git, Url, Ref}) ->
+    lock_(AppDir, {git, Url, Ref, []});
+lock_(AppDir, {git, Url, _, Opts}) ->
     AbortMsg = lists:flatten(io_lib:format("Locking of git dependency failed in ~ts", [AppDir])),
     Dir = rebar_utils:escape_double_quotes(AppDir),
     {ok, VsnString} =
         case os:type() of
             {win32, _} ->
-                rebar_utils:sh("git --git-dir=\"" ++ Dir ++ "/.git\" "
+                rebar_utils:sh("git -C \"" ++ Dir ++ "\" "
                                "--work-tree=\"" ++ Dir ++ "\" rev-parse --verify HEAD",
                     [{use_stdout, false}, {debug_abort_on_error, AbortMsg}]);
             _ ->
-                rebar_utils:sh("git --git-dir='" ++ Dir ++ "/.git' rev-parse --verify HEAD",
+                rebar_utils:sh("git -C '" ++ Dir ++ "' rev-parse --verify HEAD",
                     [{use_stdout, false}, {debug_abort_on_error, AbortMsg}])
         end,
     Ref = rebar_string:trim(VsnString, both, "\n"),
-    {git, Url, {ref, Ref}}.
+    {git, Url, {ref, Ref}, Opts}.
 
 %% Return true if either the git url or tag/branch/ref is not the same as the currently
 %% checked out git repo for the dep
 needs_update(AppInfo, _) ->
     check_type_support(),
-    needs_update_(rebar_app_info:dir(AppInfo), rebar_app_info:source(AppInfo)).
+    needs_update_(rebar_app_info:dir(AppInfo), rm_source_opts(rebar_app_info:source(AppInfo))).
 
 needs_update_(Dir, {git, Url, {tag, Tag}}) ->
     {ok, Current} = rebar_utils:sh(?FMT("git describe --tags --exact-match", []),
@@ -120,6 +128,8 @@ parse_git_url(not_scp, Url) ->
 download(TmpDir, AppInfo, State, _) ->
     check_type_support(),
     case download_(TmpDir, rebar_app_info:source(AppInfo), State) of
+        ok ->
+            ok;
         {ok, _} ->
             ok;
         {error, Reason} ->
@@ -128,29 +138,37 @@ download(TmpDir, AppInfo, State, _) ->
             {error, Error}
     end.
 
+rm_source_opts({Type, Url, Ref}) ->
+    {Type, Url, Ref};
+rm_source_opts({Type, Url, Ref, _Opts}) ->
+    {Type, Url, Ref}.
+
 %% For backward compatibilty
 download(Dir, AppInfo, State) ->
     download_(Dir, AppInfo, State).
 
 download_(Dir, {git, Url}, State) ->
     ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []),
-    download_(Dir, {git, Url, {branch, "master"}}, State);
+    download_(Dir, {git, Url, {branch, "master"}, []}, State);
 download_(Dir, {git, Url, ""}, State) ->
     ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []),
-    download_(Dir, {git, Url, {branch, "master"}}, State);
-download_(Dir, {git, Url, {branch, Branch}}, _State) ->
+    download_(Dir, {git, Url, {branch, "master"}, []}, State);
+download_(Dir, {git, Url, Checkout}, _State) ->
+    %% add an empty opts element to the tupel if there is none
+    download_(Dir, {git, Url, Checkout, []}, _State);
+download_(Dir, {git, Url, {branch, Branch}, _Opts}, _State) ->
     ok = filelib:ensure_dir(Dir),
     maybe_warn_local_url(Url),
     git_clone(branch, git_vsn(), Url, Dir, Branch);
-download_(Dir, {git, Url, {tag, Tag}}, _State) ->
+download_(Dir, {git, Url, {tag, Tag}, _Opts}, _State) ->
     ok = filelib:ensure_dir(Dir),
     maybe_warn_local_url(Url),
     git_clone(tag, git_vsn(), Url, Dir, Tag);
-download_(Dir, {git, Url, {ref, Ref}}, _State) ->
+download_(Dir, {git, Url, {ref, Ref}, _Opts}, _State) ->
     ok = filelib:ensure_dir(Dir),
     maybe_warn_local_url(Url),
     git_clone(ref, git_vsn(), Url, Dir, Ref);
-download_(Dir, {git, Url, Rev}, _State) ->
+download_(Dir, {git, Url, Rev, _Opts}, _State) ->
     ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []),
     ok = filelib:ensure_dir(Dir),
     maybe_warn_local_url(Url),
@@ -167,57 +185,60 @@ maybe_warn_local_url(Url) ->
     end.
 
 %% Use different git clone commands depending on git --version
-git_clone(branch,Vsn,Url,Dir,Branch) when Vsn >= {1,7,10}; Vsn =:= undefined ->
+git_clone(branch, GitVsn, Url, Dir, Branch) when GitVsn >= {1,7,10}; GitVsn =:= undefined ->
     rebar_utils:sh(?FMT("git clone ~ts ~ts ~ts -b ~ts --single-branch",
                         [git_clone_options(),
                          rebar_utils:escape_chars(Url),
                          rebar_utils:escape_chars(filename:basename(Dir)),
                          rebar_utils:escape_chars(Branch)]),
-                   [{cd, filename:dirname(Dir)}]);
-git_clone(branch,_Vsn,Url,Dir,Branch) ->
+                   [{cd, filename:dirname(Dir)}]),
+    ok;
+git_clone(branch, _GitVsn, Url, Dir, Branch) ->
     rebar_utils:sh(?FMT("git clone ~ts ~ts ~ts -b ~ts",
                         [git_clone_options(),
                          rebar_utils:escape_chars(Url),
                          rebar_utils:escape_chars(filename:basename(Dir)),
                          rebar_utils:escape_chars(Branch)]),
-                   [{cd, filename:dirname(Dir)}]);
-git_clone(tag,Vsn,Url,Dir,Tag) when Vsn >= {1,7,10}; Vsn =:= undefined ->
+                   [{cd, filename:dirname(Dir)}]),
+    ok;
+git_clone(tag, GitVsn, Url, Dir, Tag) when GitVsn >= {1,7,10}; GitVsn =:= undefined ->
     rebar_utils:sh(?FMT("git clone ~ts ~ts ~ts -b ~ts --single-branch",
                         [git_clone_options(),
                          rebar_utils:escape_chars(Url),
                          rebar_utils:escape_chars(filename:basename(Dir)),
                          rebar_utils:escape_chars(Tag)]),
-                   [{cd, filename:dirname(Dir)}]);
-git_clone(tag,_Vsn,Url,Dir,Tag) ->
+                   [{cd, filename:dirname(Dir)}]),
+    ok;
+git_clone(tag, _GitVsn, Url, Dir,Tag) ->
     rebar_utils:sh(?FMT("git clone ~ts ~ts ~ts -b ~ts",
                         [git_clone_options(),
                          rebar_utils:escape_chars(Url),
                          rebar_utils:escape_chars(filename:basename(Dir)),
                          rebar_utils:escape_chars(Tag)]),
-                   [{cd, filename:dirname(Dir)}]);
-git_clone(ref,_Vsn,Url,Dir,Ref) ->
+                   [{cd, filename:dirname(Dir)}]),
+    ok;
+git_clone(ref, _GitVsn, Url, Dir, Ref) ->
     rebar_utils:sh(?FMT("git clone ~ts -n ~ts ~ts",
                         [git_clone_options(),
                          rebar_utils:escape_chars(Url),
                          rebar_utils:escape_chars(filename:basename(Dir))]),
                    [{cd, filename:dirname(Dir)}]),
-    rebar_utils:sh(?FMT("git checkout -q ~ts", [Ref]), [{cd, Dir}]);
-git_clone(rev,_Vsn,Url,Dir,Rev) ->
+    rebar_utils:sh(?FMT("git checkout -q ~ts", [rebar_utils:escape_chars(Ref)]), [{cd, Dir}]);
+git_clone(rev, _GitVsn, Url, Dir, Rev) ->
     rebar_utils:sh(?FMT("git clone ~ts -n ~ts ~ts",
                         [git_clone_options(),
                          rebar_utils:escape_chars(Url),
                          rebar_utils:escape_chars(filename:basename(Dir))]),
                    [{cd, filename:dirname(Dir)}]),
-    rebar_utils:sh(?FMT("git checkout -q ~ts", [rebar_utils:escape_chars(Rev)]),
-                   [{cd, Dir}]).
+    rebar_utils:sh(?FMT("git checkout -q ~ts", [rebar_utils:escape_chars(Rev)]), [{cd, Dir}]).
 
 git_vsn() ->
     case application:get_env(rebar, git_vsn) of
-        {ok, Vsn} -> Vsn;
+        {ok, GitVsn} -> GitVsn;
         undefined ->
-            Vsn = git_vsn_fetch(),
-            application:set_env(rebar, git_vsn, Vsn),
-            Vsn
+            GitVsn = git_vsn_fetch(),
+            application:set_env(rebar, git_vsn, GitVsn),
+            GitVsn
     end.
 
 git_vsn_fetch() ->

src/rebar_git_subdir_resource.erl

@@ -0,0 +1,77 @@
+%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*-
+%% ex: ts=4 sw=4 et
+-module(rebar_git_subdir_resource).
+
+-behaviour(rebar_resource_v2).
+
+-export([init/2,
+         lock/2,
+         download/4,
+         needs_update/2,
+         make_vsn/2]).
+
+-include("rebar.hrl").
+
+%% Regex used for parsing scp style remote url
+-define(SCP_PATTERN, "\\A(?<username>[^@]+)@(?<host>[^:]+):(?<path>.+)\\z").
+
+-spec init(atom(), rebar_state:t()) -> {ok, rebar_resource_v2:resource()}.
+init(Type, _State) ->
+    Resource = rebar_resource_v2:new(Type, ?MODULE, #{}),
+    {ok, Resource}.
+
+lock(AppInfo, _) ->
+    {git_subdir, Url, Checkout, Dir} = rebar_app_info:source(AppInfo),
+    {git, Url1, {ref, Ref}, _Opts} =
+        rebar_git_resource:lock_(rebar_app_info:dir(AppInfo), {git, Url, Checkout}),
+    {git_subdir, Url1, {ref, Ref}, Dir}.
+
+download(TmpDir, AppInfo, State, _) ->
+    {git_subdir, Url, Checkout, SparseDir} = rebar_app_info:source(AppInfo),
+    case rebar_git_resource:download_(TmpDir, {git, Url, Checkout}, State) of
+        ok ->
+            sparse_checkout(rebar_git_resource:git_vsn(), TmpDir, to_ref(Checkout), SparseDir);
+        {ok, _} ->
+            sparse_checkout(rebar_git_resource:git_vsn(), TmpDir, to_ref(Checkout), SparseDir);
+        {error, Reason} ->
+            {error, Reason};
+        Error ->
+            {error, Error}
+    end.
+
+%% Return true if either the git url or tag/branch/ref is not the same as the currently
+%% checked out git repo for the dep
+needs_update(AppInfo, _) ->
+    {git_subdir, Url, Ref, _Dir} = rebar_app_info:source(AppInfo),
+    rebar_git_resource:needs_update_(rebar_app_info:dir(AppInfo), {git, Url, Ref}).
+
+make_vsn(AppInfo, _) ->
+    Dir = rebar_app_info:dir(AppInfo),
+    rebar_git_resource:make_vsn_(Dir).
+
+%%
+
+to_ref({branch, Branch}) ->
+    Branch;
+to_ref({tag, Tag}) ->
+    Tag;
+to_ref({ref, Ref}) ->
+    Ref;
+to_ref(Rev) ->
+    Rev.
+
+sparse_checkout(GitVsn, Dir, Ref, SparseDir) when GitVsn >= {1,7,4};
+                                                  GitVsn =:= undefined  ->
+    ?DEBUG("doing sparse checkout in ~s of dir ~s", [Dir, SparseDir]),
+
+    rebar_utils:sh(?FMT("git --git-dir=.git config core.sparsecheckout true", []),
+                   [{cd, Dir}]),
+    filelib:ensure_dir(filename:join(Dir, ".git/info/sparse-checkout")),
+    file:write_file(filename:join(Dir, ".git/info/sparse-checkout"), SparseDir),
+    rebar_utils:sh(?FMT("git checkout -q ~ts", [rebar_utils:escape_chars(Ref)]), [{cd, Dir}]),
+    ok;
+sparse_checkout(_, _, _, _) ->
+    %% sparse checkout not supported but we can still use the subdirectory
+    %% so no need to fail, just don't do the sparse checkout
+    ?DEBUG("too old a git version to do a sparse checkout for a subdir dep", []),
+    ok.

src/rebar_prv_path.erl

@@ -12,7 +12,7 @@
 -include("rebar.hrl").
 
 -define(PROVIDER, path).
--define(DEPS, [app_discovery]).
+-define(DEPS, [compile]).
 
 %% ===================================================================
 %% Public API
@@ -54,11 +54,11 @@ filter_apps(RawOpts, State) ->
         [] ->
             ProjectDeps = project_deps(State),
             ProjectApps = rebar_state:project_apps(State),
-            lists:map(fun(A) -> binary_to_list(rebar_app_info:name(A)) end, ProjectApps) ++ ProjectDeps;
-        _  -> Apps
+            ProjectApps ++ ProjectDeps;
+        _  ->
+            Apps
     end.
 
-
 paths([], _, State, Acc) -> print_paths_if_exist(lists:reverse(Acc), State);
 paths([{base, true}|Rest], Apps, State, Acc) ->
     paths(Rest, Apps, State, [base_dir(State)|Acc]);
@@ -80,12 +80,12 @@ bin_dir(State)  -> io_lib:format("~ts/bin", [rebar_dir:base_dir(State)]).
 lib_dir(State)  -> io_lib:format("~ts", [rebar_dir:deps_dir(State)]).
 rel_dir(State)  -> io_lib:format("~ts/rel", [rebar_dir:base_dir(State)]).
 
-ebin_dirs(Apps, State) ->
-    lists:map(fun(App) -> io_lib:format("~ts/~ts/ebin", [rebar_dir:deps_dir(State), App]) end, Apps).
-priv_dirs(Apps, State) ->
-    lists:map(fun(App) -> io_lib:format("~ts/~ts/priv", [rebar_dir:deps_dir(State), App]) end, Apps).
-src_dirs(Apps, State) ->
-    lists:map(fun(App) -> io_lib:format("~ts/~ts/src", [rebar_dir:deps_dir(State), App]) end, Apps).
+ebin_dirs(Apps, _State) ->
+    lists:map(fun(App) -> rebar_app_info:ebin_dir(App) end, Apps).
+priv_dirs(Apps, _State) ->
+    lists:map(fun(App) -> rebar_app_info:priv_dir(App) end, Apps).
+src_dirs(Apps, _State) ->
+    lists:map(fun(App) -> filename:join(rebar_app_info:out_dir(App), "src") end, Apps).
 
 print_paths_if_exist(Paths, State) ->
     {RawOpts, _} = rebar_state:command_parsed_args(State),
@@ -94,18 +94,7 @@ print_paths_if_exist(Paths, State) ->
     io:format("~ts", [rebar_string:join(RealPaths, Sep)]).
 
 project_deps(State) ->
-    Profiles = rebar_state:current_profiles(State),
-    DepList = lists:foldl(fun(Profile, Acc) -> rebar_state:get(State, {deps, Profile}, []) ++ Acc end, [], Profiles),
-    LockList = lists:foldl(fun(Profile, Acc) -> rebar_state:get(State, {locks, Profile}, []) ++ Acc end, [], Profiles),
-    Deps = [normalize(name(Dep)) || Dep <- DepList++LockList],
-    lists:usort(Deps).
-
-name(App) when is_tuple(App) -> element(1, App);
-name(Name) when is_binary(Name); is_list(Name); is_atom(Name) -> Name.
-
-normalize(AppName) when is_list(AppName) -> AppName;
-normalize(AppName) when is_atom(AppName) -> atom_to_list(AppName);
-normalize(AppName) when is_binary(AppName) -> binary_to_list(AppName).
+    rebar_state:all_deps(State).
 
 path_opts(_State) ->
     [{app, undefined, "app", string, help(app)},

test/mock_git_subdir_resource.erl

@@ -0,0 +1,142 @@
+%%% Mock a git_subdir resource and create an app magically for each
+%%% URL and subdirectory submitted.
+-module(mock_git_subdir_resource).
+-export([mock/0, mock/1, mock/2, unmock/0]).
+-define(MOD, rebar_git_subdir_resource).
+
+%%%%%%%%%%%%%%%%%
+%%% Interface %%%
+%%%%%%%%%%%%%%%%%
+
+%% @doc same as `mock([])'.
+mock() -> mock([]).
+
+%% @doc Mocks a fake version of the git resource fetcher that creates
+%% empty applications magically, rather than trying to download them.
+%% Specific config options are explained in each of the private functions.
+-spec mock(Opts) -> ok when
+    Opts :: [Option],
+    Option :: {update, [App]}
+            | {default_vsn, Vsn}
+            | {override_vsn, [{App, Vsn}]}
+            | {deps, [{App, [Dep]}]},
+    App :: string(),
+    Dep :: {App, {git_subdir, string(), term(), string()}}
+         | {pkg, App, term()},
+    Vsn :: string().
+mock(Opts) ->
+    mock(Opts, create_app).
+
+mock(Opts, CreateType) ->
+    meck:new(?MOD, [no_link, passthrough]),
+    mock_lock(Opts),
+    mock_update(Opts),
+    mock_vsn(Opts),
+    mock_download(Opts, CreateType),
+    ok.
+
+unmock() ->
+    meck:unload(?MOD).
+
+%%%%%%%%%%%%%%%
+%%% Private %%%
+%%%%%%%%%%%%%%%
+
+%% @doc creates values for a lock file. The refs are fake, but
+%% tags and existing refs declared for a dependency are preserved.
+mock_lock(_) ->
+    meck:expect(
+        ?MOD, lock,
+        fun(AppInfo, _) ->
+            case rebar_app_info:source(AppInfo) of
+                {git_subdir, Url, {tag, Ref}, Dir} -> {git_subdir, Url, {ref, Ref}, Dir};
+                {git_subdir, Url, {ref, Ref}, Dir} -> {git_subdir, Url, {ref, Ref}, Dir};
+                {git_subdir, Url, Dir} -> {git_subdir, Url, {ref, "0.0.0"}, Dir};
+                {git_subdir, Url, _, Dir} -> {git_subdir, Url, {ref, "0.0.0"}, Dir}
+            end
+        end).
+
+%% @doc The config passed to the `mock/2' function can specify which apps
+%% should be updated on a per-name basis: `{update, ["App1", "App3"]}'.
+mock_update(Opts) ->
+    ToUpdate = proplists:get_value(upgrade, Opts, []),
+%    ct:pal("TOUp: ~p", [ToUpdate]),
+    meck:expect(
+        ?MOD, needs_update,
+        fun(AppInfo, _) ->
+            {git_subdir, Url, _Ref} = rebar_app_info:source(AppInfo),
+            App = app(Url),
+%            ct:pal("Needed update? ~p (~p) -> ~p", [App, {Url,_Ref}, lists:member(App, ToUpdate)]),
+            lists:member(App, ToUpdate)
+        end).
+
+%% @doc Tries to fetch a version from the `*.app.src' file or otherwise
+%% just returns random stuff, avoiding to check for the presence of git.
+%% This probably breaks the assumption that stable references are returned.
+%%
+%% This function can't respect the `override_vsn' option because if the
+%% .app.src file isn't there, we can't find the app name either.
+mock_vsn(Opts) ->
+    Default = proplists:get_value(default_vsn, Opts, "0.0.0"),
+    meck:expect(
+        ?MOD, make_vsn,
+      fun(AppInfo, _) ->
+            Dir = rebar_app_info:dir(AppInfo),
+            case filelib:wildcard("*.app.src", filename:join([Dir,"src"])) of
+                [AppSrc] ->
+                    {ok, App} = file:consult(AppSrc),
+                    Vsn = proplists:get_value(vsn, App),
+                    {plain, Vsn};
+                _ ->
+                    {plain, Default}
+            end
+        end).
+
+%% @doc For each app to download, create a dummy app on disk instead.
+%% The configuration for this one (passed in from `mock/1') includes:
+%%
+%% - Specify a version, branch, ref, or tag via the `{git_subdir, URL, {_, Vsn}'
+%%   format to specify a path.
+%% - If there is no version submitted (`{git_subdir, URL}'), the function instead
+%%   reads from the `override_vsn' proplist (`{override_vsn, {"App1","1.2.3"}'),
+%%   and otherwise uses the value associated with `default_vsn'.
+%% - Dependencies for each application must be passed of the form:
+%%   `{deps, [{"app1", [{app2, ".*", {git_subdir, ...}}]}]}' -- basically
+%%   the `deps' option takes a key/value list of terms to output directly
+%%   into a `rebar.config' file to describe dependencies.
+mock_download(Opts, CreateType) ->
+    Deps = proplists:get_value(deps, Opts, []),
+    Config = proplists:get_value(config, Opts, []),
+    Default = proplists:get_value(default_vsn, Opts, "0.0.0"),
+    Overrides = proplists:get_value(override_vsn, Opts, []),
+    meck:expect(
+        ?MOD, download,
+        fun (Dir, AppInfo, _, _) ->
+            Git = rebar_app_info:source(AppInfo),
+            filelib:ensure_dir(Dir),
+            {git_subdir, Url, {_, Vsn}, SubDir} = normalize_git(Git, Overrides, Default),
+            FullSubDir = filename:join(Dir, SubDir),
+            filelib:ensure_dir(FullSubDir),
+            App = app(Url),
+            AppDeps = proplists:get_value({App,Vsn}, Deps, []),
+            rebar_test_utils:CreateType(
+                FullSubDir, App, Vsn,
+                [kernel, stdlib] ++ [element(1,D) || D  <- AppDeps]
+            ),
+            rebar_test_utils:create_config(FullSubDir, [{deps, AppDeps}]++Config),
+            ok
+        end).
+
+%%%%%%%%%%%%%%%
+%%% Helpers %%%
+%%%%%%%%%%%%%%%
+app(Path) ->
+    filename:basename(Path, ".git").
+
+normalize_git({git_subdir, Url, SubDir}, Overrides, Default) ->
+    Vsn = proplists:get_value(app(Url), Overrides, Default),
+    {git, Url, {tag, Vsn}, SubDir};
+normalize_git({git_subdir, Url, Branch, SubDir}, _, _) when is_list(Branch) ->
+    {git, Url, {branch, Branch}, SubDir};
+normalize_git(Git, _, _) ->
+    Git.

test/rebar_compile_SUITE.erl

@@ -27,7 +27,7 @@ all() ->
      parse_transform_test, erl_first_files_test, mib_test,
      umbrella_mib_first_test, only_default_transitive_deps, clean_all,
      clean_specific, profile_deps, deps_build_in_prod, only_deps,
-     override_deps, override_add_deps, override_del_deps,
+     override_deps, git_subdir_deps, override_add_deps, override_del_deps,
      override_opts, override_add_opts, override_del_opts,
      apply_overrides_exactly_once, override_only_deps,
      profile_override_deps, profile_override_add_deps, profile_override_del_deps,
@@ -1531,6 +1531,22 @@ override_deps(Config) ->
               {dep_not_exist, "other_dep"}]}
     ).
 
+git_subdir_deps(Config) ->
+    Deps = rebar_test_utils:expand_deps(git_subdir, [{"some_dep", "0.0.1", [{"other_dep", "0.0.1", []}]}]),
+    TopDeps = rebar_test_utils:top_level_deps(Deps),
+
+    {SrcDeps, _} = rebar_test_utils:flat_deps(Deps),
+    mock_git_subdir_resource:mock([{deps, SrcDeps}]),
+
+    RebarConfig = [
+        {deps, TopDeps}
+    ],
+    rebar_test_utils:run_and_check(
+        Config, RebarConfig, ["compile"],
+        {ok, [{subdir_dep, "some_dep"},
+              {subdir_dep, "other_dep"}]}
+    ).
+
 override_add_deps(Config) ->
     Deps = rebar_test_utils:expand_deps(git, [{"some_dep", "0.0.1", [{"other_dep", "0.0.1", []}]}]),
     TopDeps = rebar_test_utils:top_level_deps(Deps),

test/rebar_test_utils.erl

@@ -183,6 +183,12 @@ random_seed() ->
 -endif.
 
 expand_deps(_, []) -> [];
+expand_deps(git_subdir, [{Name, Deps} | Rest]) ->
+    Dep = {Name, {git_subdir, "https://example.org/user/"++Name++".git", {branch, "master"}, "appsubdir"}},
+    [{Dep, expand_deps(git_subdir, Deps)} | expand_deps(git_subdir, Rest)];
+expand_deps(git_subdir, [{Name, Vsn, Deps} | Rest]) ->
+    Dep = {Name, Vsn, {git_subdir, "https://example.org/user/"++Name++".git", {tag, Vsn}, "appsubdir"}},
+    [{Dep, expand_deps(git_subdir, Deps)} | expand_deps(git_subdir, Rest)];
 expand_deps(git, [{Name, Deps} | Rest]) ->
     Dep = {Name, ".*", {git, "https://example.org/user/"++Name++".git", "master"}},
     [{Dep, expand_deps(git, Deps)} | expand_deps(git, Rest)];
@@ -226,6 +232,12 @@ flat_deps([{{pkg, Name, Vsn, undefined, undefined}, PkgDeps} | Rest], Src, Pkg)
               Src,
               Pkg ++ [Current | FlatPkgDeps]);
 flat_deps([{{Name,_Vsn,Ref}, Deps} | Rest], Src, Pkg) ->
+    Current = {{Name,vsn_from_ref(Ref)}, top_level_deps(Deps)},
+    {FlatDeps, FlatPkgDeps} = flat_deps(Deps),
+    flat_deps(Rest,
+              Src ++ [Current | FlatDeps],
+              Pkg ++ FlatPkgDeps);
+flat_deps([{{Name,Ref}, Deps} | Rest], Src, Pkg) ->
     Current = {{Name,vsn_from_ref(Ref)}, top_level_deps(Deps)},
     {FlatDeps, FlatPkgDeps} = flat_deps(Deps),
     flat_deps(Rest,
@@ -233,7 +245,9 @@ flat_deps([{{Name,_Vsn,Ref}, Deps} | Rest], Src, Pkg) ->
               Pkg ++ FlatPkgDeps).
 
 vsn_from_ref({git, _, {_, Vsn}}) -> Vsn;
-vsn_from_ref({git, _, Vsn}) -> Vsn.
+vsn_from_ref({git, _, Vsn}) -> Vsn;
+vsn_from_ref({git_subdir, _, {_, Vsn}, _}) -> Vsn;
+vsn_from_ref({git_subdir, _, Vsn, _}) -> Vsn.
 
 top_level_deps([]) -> [];
 top_level_deps([{{pkg, Name, Vsn, undefined, undefined}, _} | Deps]) ->
@@ -246,6 +260,8 @@ top_level_deps([{{Name, Vsn, Ref}, _} | Deps]) ->
 %%%%%%%%%%%%%%%
 check_results(AppDir, Expected, ProfileRun) ->
     BuildDirs = filelib:wildcard(filename:join([AppDir, "_build", ProfileRun, "lib", "*"])),
+    BuildSubDirs = [D || D <- filelib:wildcard(filename:join([AppDir, "_build", ProfileRun, "lib", "*", "*"])),
+                         filelib:is_dir(D)],
     PluginDirs = filelib:wildcard(filename:join([AppDir, "_build", ProfileRun, "plugins", "*"])),
     GlobalPluginDirs = filelib:wildcard(filename:join([AppDir, "global", "plugins", "*"])),
     CheckoutsDirs = filelib:wildcard(filename:join([AppDir, "_checkouts", "*"])),
@@ -259,7 +275,9 @@ check_results(AppDir, Expected, ProfileRun) ->
     ValidDepsNames = [{ec_cnv:to_list(rebar_app_info:name(App)), App} || App <- ValidApps],
 
     Deps = rebar_app_discover:find_apps(BuildDirs, all),
+    SubDeps = rebar_app_discover:find_apps(BuildSubDirs, all),
     DepsNames = [{ec_cnv:to_list(rebar_app_info:name(App)), App} || App <- Deps],
+    SubDirDepsNames = [{ec_cnv:to_list(rebar_app_info:name(App)), App} || App <- SubDeps],
     Checkouts = rebar_app_discover:find_apps(CheckoutsDirs, all),
     CheckoutsNames = [{ec_cnv:to_list(rebar_app_info:name(App)), App} || App <- Checkouts],
     Plugins = rebar_app_discover:find_apps(PluginDirs, all),
@@ -323,6 +341,18 @@ check_results(AppDir, Expected, ProfileRun) ->
                         ?assertEqual(iolist_to_binary(Vsn),
                                      iolist_to_binary(rebar_app_info:original_vsn(App)))
                 end
+        ;  ({subdir_dep, Name}) ->
+                ct:pal("Subdir Dep Name: ~p", [Name]),
+                ?assertNotEqual(false, lists:keyfind(Name, 1, SubDirDepsNames))
+        ;  ({subdir_dep, Name, Vsn}) ->
+                ct:pal("Subdir Dep Name: ~p, Vsn: ~p", [Name, Vsn]),
+                case lists:keyfind(Name, 1, SubDirDepsNames) of
+                    false ->
+                        error({dep_not_found, Name});
+                    {Name, App} ->
+                        ?assertEqual(iolist_to_binary(Vsn),
+                                     iolist_to_binary(rebar_app_info:original_vsn(App)))
+                end
         ;  ({plugin, Name}) ->
                 ct:pal("Plugin Name: ~p", [Name]),
                 ?assertNotEqual(false, lists:keyfind(Name, 1, PluginsNames))