Merge pull request #286 from tsloughter/checkouts

treat _checkouts as deps that are always compiled

src/rebar.hrl

@@ -14,7 +14,9 @@
 -define(FMT(Str, Args), lists:flatten(io_lib:format(Str, Args))).
 
 -define(DEFAULT_BASE_DIR, "_build").
--define(DEFAULT_PROJECT_APP_DIRS, ["_checkouts", "apps", "lib", "."]).
+-define(DEFAULT_ROOT_DIR, ".").
+-define(DEFAULT_PROJECT_APP_DIRS, ["apps", "lib", "."]).
+-define(DEFAULT_CHECKOUTS_DIR, "_checkouts").
 -define(DEFAULT_DEPS_DIR, "lib").
 -define(DEFAULT_PLUGINS_DIR, "plugins").
 -define(DEFAULT_TEST_DEPS_DIR, "test/lib").

src/rebar_app_info.erl

@@ -34,6 +34,8 @@
          source/2,
          state/1,
          state/2,
+         is_checkout/1,
+         is_checkout/2,
          valid/1,
          valid/2]).
 
@@ -53,6 +55,7 @@
                      out_dir :: file:name(),
                      source :: string() | tuple() | undefined,
                      state :: rebar_state:t() | undefined,
+                     is_checkout=false :: boolean(),
                      valid :: boolean()}).
 
 %%============================================================================
@@ -238,6 +241,14 @@ state(AppInfo=#app_info_t{}, State) ->
 state(#app_info_t{state=State}) ->
     State.
 
+-spec is_checkout(t(), boolean()) -> t().
+is_checkout(AppInfo=#app_info_t{}, IsCheckout) ->
+    AppInfo#app_info_t{is_checkout=IsCheckout}.
+
+-spec is_checkout(t()) -> boolean().
+is_checkout(#app_info_t{is_checkout=IsCheckout}) ->
+    IsCheckout.
+
 -spec valid(t()) -> boolean().
 valid(AppInfo=#app_info_t{valid=undefined}) ->
     case rebar_app_utils:validate_application_info(AppInfo) of

src/rebar_dir.erl

@@ -3,6 +3,8 @@
 -export([base_dir/1,
          deps_dir/1,
          deps_dir/2,
+         checkouts_dir/1,
+         checkouts_dir/2,
          plugins_dir/1,
          lib_dirs/1,
          home_dir/0,
@@ -40,6 +42,17 @@ deps_dir(State) ->
 deps_dir(DepsDir, App) ->
     filename:join(DepsDir, App).
 
+root_dir(State) ->
+    rebar_state:get(State, root_dir, ?DEFAULT_ROOT_DIR).
+
+-spec checkouts_dir(rebar_state:t()) -> file:filename_all().
+checkouts_dir(State) ->
+    filename:join(root_dir(State), rebar_state:get(State, checkouts_dir, ?DEFAULT_CHECKOUTS_DIR)).
+
+-spec checkouts_dir(rebar_state:t(), file:filename_all()) -> file:filename_all().
+checkouts_dir(State, App) ->
+    filename:join(checkouts_dir(State), App).
+
 -spec plugins_dir(rebar_state:t()) -> file:filename_all().
 plugins_dir(State) ->
     case lists:member(global, rebar_state:current_profiles(State)) of

src/rebar_prv_install_deps.erl

@@ -174,7 +174,7 @@ compile_order(Source, ProjectApps) ->
     case rebar_digraph:compile_order(Source) of
         {ok, Sort} ->
             %% Valid apps are compiled and good
-            {ok, lists:dropwhile(fun rebar_app_info:valid/1, Sort -- ProjectApps)};
+            {ok, lists:dropwhile(fun not_needs_compile/1, Sort -- ProjectApps)};
         {error, Error} ->
             {error, Error}
     end.
@@ -214,23 +214,28 @@ handle_pkg_dep(Profile, Pkg, Packages, Upgrade, DepsDir, Fetched, Seen, State) -
     {[AppInfo | Fetched], NewSeen, NewState}.
 
 maybe_lock(Profile, AppInfo, Seen, State, Level) ->
-    case Profile of
-        default ->
-            Name = rebar_app_info:name(AppInfo),
-            case sets:is_element(Name, Seen) of
-                false ->
-                    Locks = rebar_state:lock(State),
-                    case lists:any(fun(App) -> rebar_app_info:name(App) =:= Name end, Locks) of
-                        true ->
-                            {sets:add_element(Name, Seen), State};
+    case rebar_app_info:is_checkout(AppInfo) of
+        false ->
+            case Profile of
+                default ->
+                    Name = rebar_app_info:name(AppInfo),
+                    case sets:is_element(Name, Seen) of
                         false ->
-                            {sets:add_element(Name, Seen),
-                             rebar_state:lock(State, rebar_app_info:dep_level(AppInfo, Level))}
+                            Locks = rebar_state:lock(State),
+                            case lists:any(fun(App) -> rebar_app_info:name(App) =:= Name end, Locks) of
+                                true ->
+                                    {sets:add_element(Name, Seen), State};
+                                false ->
+                                    {sets:add_element(Name, Seen),
+                                     rebar_state:lock(State, rebar_app_info:dep_level(AppInfo, Level))}
+                            end;
+                        true ->
+                            {Seen, State}
                     end;
-                true ->
+                _ ->
                     {Seen, State}
             end;
-        _ ->
+        true ->
             {Seen, State}
     end.
 
@@ -367,7 +372,7 @@ handle_dep(State, DepsDir, AppInfo, Locks, Level) ->
 maybe_fetch(AppInfo, Upgrade, Seen, State) ->
     AppDir = ec_cnv:to_list(rebar_app_info:dir(AppInfo)),
     %% Don't fetch dep if it exists in the _checkouts dir
-    case in_checkouts(AppInfo) of
+    case 
rebar_app_info:is_checkout(AppInfo) of
         true ->
             false;
         false ->
@@ -392,13 +397,6 @@ maybe_fetch(AppInfo, Upgrade, Seen, State) ->
             end
     end.
 
-in_checkouts(AppInfo) ->
-    Apps = rebar_app_discover:find_apps(["_checkouts"], all),
-    case rebar_app_utils:find(rebar_app_info:name(AppInfo), Apps) of
-        {ok, _} -> true;
-        error -> false
-    end.
-
 in_default(AppInfo, State) ->
     Name = ec_cnv:to_list(rebar_app_info:name(AppInfo)),
     DefaultAppDir = filename:join([rebar_state:get(State, base_dir), "default", "lib", Name]),
@@ -427,11 +425,25 @@ parse_deps(DepsDir, Deps, State, Locks, Level) ->
                         end
                 end, {[], []}, Deps).
 
-parse_dep({Name, Vsn}, {SrcDepsAcc, PkgDepsAcc}, _DepsDir, _State) when is_list(Vsn) ->
-    {SrcDepsAcc, [parse_goal(ec_cnv:to_binary(Name)
-                            ,ec_cnv:to_binary(Vsn)) | PkgDepsAcc]};
-parse_dep(Name, {SrcDepsAcc, PkgDepsAcc}, _DepsDir, _State) when is_atom(Name) ->
-    {SrcDepsAcc, [ec_cnv:to_binary(Name) | PkgDepsAcc]};
+parse_dep({Name, Vsn}, {SrcDepsAcc, PkgDepsAcc}, DepsDir, State) when is_list(Vsn) ->
+    CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)),
+    case rebar_app_info:discover(CheckoutsDir) of
+        {ok, _App} ->
+            Dep = new_dep(DepsDir, Name, [], [], State),
+            {[Dep | SrcDepsAcc], PkgDepsAcc};
+        not_found ->
+            {SrcDepsAcc, [parse_goal(ec_cnv:to_binary(Name)
+                                    ,ec_cnv:to_binary(Vsn)) | PkgDepsAcc]}
+    end;
+parse_dep(Name, {SrcDepsAcc, PkgDepsAcc}, DepsDir, State) when is_atom(Name) ->
+    CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)),
+    case rebar_app_info:discover(CheckoutsDir) of
+        {ok, _App} ->
+            Dep = new_dep(DepsDir, Name, [], [], State),
+            {[Dep | SrcDepsAcc], PkgDepsAcc};
+        not_found ->
+            {SrcDepsAcc, [ec_cnv:to_binary(Name) | PkgDepsAcc]}
+    end;
 parse_dep({Name, Source}, {SrcDepsAcc, PkgDepsAcc}, DepsDir, State) when is_tuple(Source) ->
     Dep = new_dep(DepsDir, Name, [], Source, State),
     {[Dep | SrcDepsAcc], PkgDepsAcc};
@@ -445,8 +457,15 @@ parse_dep({Name, _Vsn, Source, Opts}, {SrcDepsAcc, PkgDepsAcc}, DepsDir, State)
     ?WARN("Dependency option list ~p in ~p is not supported and will be ignored", [Opts, Name]),
     Dep = new_dep(DepsDir, Name, [], Source, State),
     {[Dep | SrcDepsAcc], PkgDepsAcc};
-parse_dep({_Name, {pkg, Name, Vsn}, Level}, {SrcDepsAcc, PkgDepsAcc}, _, _) when is_integer(Level) ->
-    {SrcDepsAcc, [{Name, Vsn} | PkgDepsAcc]};
+parse_dep({_Name, {pkg, Name, Vsn}, Level}, {SrcDepsAcc, PkgDepsAcc}, DepsDir, State) when is_integer(Level) ->
+    CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)),
+    case rebar_app_info:discover(CheckoutsDir) of
+        {ok, _App} ->
+            Dep = new_dep(DepsDir, Name, [], [], State),
+            {[Dep | SrcDepsAcc], PkgDepsAcc};
+        not_found ->
+            {SrcDepsAcc, [{Name, Vsn} | PkgDepsAcc]}
+    end;
 parse_dep({Name, Source, Level}, {SrcDepsAcc, PkgDepsAcc}, DepsDir, State) when is_tuple(Source)
                                                                               , is_integer(Level) ->
     Dep = new_dep(DepsDir, Name, [], Source, State),
@@ -456,12 +475,19 @@ parse_dep(Dep, _, _, _) ->
 
 
 new_dep(DepsDir, Name, Vsn, Source, State) ->
-    Dir = ec_cnv:to_list(filename:join(DepsDir, Name)),
-    {ok, Dep} = case rebar_app_info:discover(Dir) of
+    CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)),
+    {ok, Dep} = case rebar_app_info:discover(CheckoutsDir) of
                     {ok, App} ->
-                        {ok, App};
+                        {ok, rebar_app_info:is_checkout(App, true)};
                     not_found ->
-                        rebar_app_info:new(Name, Vsn, ec_cnv:to_list(filename:join(DepsDir, Name)))
+                        Dir = ec_cnv:to_list(filename:join(DepsDir, Name)),
+                        case rebar_app_info:discover(Dir) of
+                            {ok, App} ->
+                                {ok, App};
+                            not_found ->
+                                rebar_app_info:new(Name, Vsn,
+                                                   ec_cnv:to_list(filename:join(DepsDir, Name)))
+                        end
                 end,
     C = rebar_config:consult(rebar_app_info:dir(Dep)),
     S = rebar_state:new(rebar_state:new(), C, rebar_app_info:dir(Dep)),
@@ -528,3 +554,7 @@ warn_skip_pkg({Name, Source}, State) ->
         false -> ?WARN(Msg, Args);
         true -> ?ERROR(Msg, Args), ?FAIL
     end.
+
+not_needs_compile(App) ->
+    not(rebar_app_info:is_checkout(App))
+        andalso rebar_app_info:valid(App).

src/rebar_prv_lock.erl

@@ -30,16 +30,16 @@ init(State) ->
 -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}.
 do(State) ->
     AllDeps = rebar_state:lock(State),
-    Locks = lists:map(fun(Dep) ->
-                              Dir = rebar_app_info:dir(Dep),
-                              Source = rebar_app_info:source(Dep),
-
-                              %% If source is tuple it is a source dep
-                              %% e.g. {git, "git://github.com/ninenines/cowboy.git", "master"}
-                              {rebar_app_info:name(Dep)
-                              ,rebar_fetch:lock_source(Dir, Source)
-                              ,rebar_app_info:dep_level(Dep)}
-                      end, AllDeps),
+    Locks = [begin
+                 Dir = rebar_app_info:dir(Dep),
+                 Source = rebar_app_info:source(Dep),
+
+                 %% If source is tuple it is a source dep
+                 %% e.g. {git, "git://github.com/ninenines/cowboy.git", "master"}
+                 {rebar_app_info:name(Dep)
+                 ,rebar_fetch:lock_source(Dir, Source)
+                 ,rebar_app_info:dep_level(Dep)}
+             end || Dep <- AllDeps, not(rebar_app_info:is_checkout(Dep))],
     Dir = rebar_state:dir(State),
     file:write_file(filename:join(Dir, ?LOCK_FILE), io_lib:format("~p.~n", [Locks])),
     {ok, State}.

src/rebar_utils.erl

@@ -387,6 +387,7 @@ beams(Dir) ->
 -spec abort() -> no_return().
 abort() ->
     throw(rebar_abort).
+
 -spec abort(string(), [term()]) -> no_return().
 abort(String, Args) ->
     ?ERROR(String, Args),

test/mock_git_resource.erl

@@ -55,7 +55,7 @@ mock_lock(_) ->
 %% should be updated on a per-name basis: `{update, ["App1", "App3"]}'.
 mock_update(Opts) ->
     ToUpdate = proplists:get_value(upgrade, Opts, []),
-    ct:pal("TOUp: ~p", [ToUpdate]),
+%    ct:pal("TOUp: ~p", [ToUpdate]),
     meck:expect(
         ?MOD, needs_update,
         fun(_Dir, {git, Url, _Ref}) ->
@@ -110,7 +110,7 @@ mock_download(Opts) ->
             AppDeps = proplists:get_value({App,Vsn}, Deps, []),
             rebar_test_utils:create_app(
                 Dir, App, Vsn,
-                [element(1,D) || D  <- AppDeps]
+                [kernel, stdlib] ++ [element(1,D) || D  <- AppDeps]
             ),
             rebar_test_utils:create_config(Dir, [{deps, AppDeps}]),
             {ok, 'WHATEVER'}

test/mock_pkg_resource.erl

@@ -78,9 +78,9 @@ mock_download(Opts) ->
             App = binary_to_list(AppBin),
             filelib:ensure_dir(Dir),
             AppDeps = proplists:get_value({App,Vsn}, Deps, []),
-            {ok, AppInfo} = rebar_test_utils:create_empty_app(
-                Dir, App, Vsn,
-                [element(1,D) || D  <- AppDeps]
+            {ok, AppInfo} = rebar_test_utils:create_app(
+                Dir, App, binary_to_list(Vsn),
+                [kernel, stdlib] ++ [element(1,D) || D  <- AppDeps]
             ),
             rebar_test_utils:create_config(Dir, [{deps, AppDeps}]),
             Tarball = filename:join([Dir, App++"-"++binary_to_list(Vsn)++".tar"]),

test/rebar_compile_SUITE.erl

@@ -4,6 +4,7 @@
          init_per_suite/1,
          end_per_suite/1,
          init_per_testcase/2,
+         end_per_testcase/2,
          all/0,
          build_basic_app/1,
          build_release_apps/1,
@@ -12,7 +13,9 @@
          build_all_srcdirs/1,
          recompile_when_opts_change/1,
          dont_recompile_when_opts_dont_change/1,
-         dont_recompile_yrl_or_xrl/1]).
+         dont_recompile_yrl_or_xrl/1,
+         deps_in_path/1,
+         checkout_priority/1]).
 
 -include_lib("common_test/include/ct.hrl").
 -include_lib("eunit/include/eunit.hrl").
@@ -30,12 +33,15 @@ end_per_suite(_Config) ->
 init_per_testcase(_, Config) ->
     rebar_test_utils:init_rebar_state(Config).
 
+end_per_testcase(_, _Config) ->
+    catch meck:unload().
+
 all() ->
     [build_basic_app, build_release_apps,
      build_checkout_apps, build_checkout_deps,
      build_all_srcdirs,
      recompile_when_opts_change, dont_recompile_when_opts_dont_change,
-     dont_recompile_yrl_or_xrl].
+     dont_recompile_yrl_or_xrl, deps_in_path, checkout_priority].
 
 build_basic_app(Config) ->
     AppDir = ?config(apps, Config),
@@ -88,8 +94,11 @@ build_checkout_deps(Config) ->
     rebar_test_utils:create_app(filename:join([CheckoutsDir,Name2]), Name2, Vsn2, [kernel, stdlib]),
     rebar_test_utils:create_app(filename:join([DepsDir,Name2]), Name2, Vsn1, [kernel, stdlib]),
 
+    Deps = [{list_to_atom(Name2), Vsn2, {git, "", ""}}],
+    {ok, RebarConfig} = file:consult(rebar_test_utils:create_config(AppDir, [{deps, Deps}])),
+
     rebar_test_utils:run_and_check(
-        Config, [], ["compile"],
+        Config, RebarConfig, ["compile"],
         {ok, [{app, Name1}, {checkout, Name2}]}
     ),
     ok = application:load(list_to_atom(Name2)),
@@ -211,3 +220,115 @@ dont_recompile_yrl_or_xrl(Config) ->
 
     ?assert(ModTime == NewModTime).
 
+deps_in_path(Config) ->
+    AppDir = ?config(apps, Config),
+    StartPaths = code:get_path(),
+
+    Name = rebar_test_utils:create_random_name("app1_"),
+    Vsn = rebar_test_utils:create_random_vsn(),
+    rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]),
+
+    DepName = rebar_test_utils:create_random_name("dep1_"),
+    PkgName = rebar_test_utils:create_random_name("pkg1_"),
+    mock_git_resource:mock([]),
+    mock_pkg_resource:mock([
+        {pkgdeps, [{{iolist_to_binary(PkgName), iolist_to_binary(Vsn)}, []}]}
+    ]),
+
+    RConfFile = rebar_test_utils:create_config(AppDir, [{deps, [
+        {list_to_atom(DepName), {git, "http://site.com/user/"++DepName++".git", {tag, Vsn}}},
+        {list_to_atom(PkgName), Vsn}
+    ]}]),
+    {ok, RConf} = file:consult(RConfFile),
+    %% Make sure apps we look for are not visible
+    %% Hope not to find src name
+    ?assertEqual([], [Path || Path <- code:get_path(),
+                              {match, _} <- [re:run(Path, DepName)]]),
+    %% Hope not to find pkg name in there
+    ?assertEqual([], [Path || Path <- code:get_path(),
+                                 {match, _} <- [re:run(Path, PkgName)]]),
+    %% Build things
+    rebar_test_utils:run_and_check(
+        Config, RConf, ["compile"],
+        {ok, [{app, Name}, {dep, DepName}, {dep, PkgName}]}
+    ),
+    %% Find src name in there
+    ?assertNotEqual([], [Path || Path <- code:get_path(),
+                                 {match, _} <- [re:run(Path, DepName)]]),
+    %% find pkg name in there
+    ?assertNotEqual([], [Path || Path <- code:get_path(),
+                                 {match, _} <- [re:run(Path, PkgName)]]),
+    code:set_path(StartPaths),
+    %% Make sure apps we look for are not visible again
+    %% Hope not to find src name
+    ?assertEqual([], [Path || Path <- code:get_path(),
+                              {match, _} <- [re:run(Path, DepName)]]),
+    %% Hope not to find pkg name in there
+    ?assertEqual([], [Path || Path <- code:get_path(),
+                                 {match, _} <- [re:run(Path, PkgName)]]),
+    %% Rebuild
+    rebar_test_utils:run_and_check(
+        Config, RConf, ["compile"],
+        {ok, [{app, Name}, {dep, DepName}, {dep, PkgName}]}
+    ),
+    %% Find src name in there
+    ?assertNotEqual([], [Path || Path <- code:get_path(),
+                                 {match, _} <- [re:run(Path, DepName)]]),
+    %% find pkg name in there
+    ?assertNotEqual([], [Path || Path <- code:get_path(),
+                                 {match, _} <- [re:run(Path, PkgName)]]).
+
+checkout_priority(Config) ->
+    AppDir = ?config(apps, Config),
+    CheckoutsDir = ?config(checkouts, Config),
+    StartPaths = code:get_path(),
+
+    Name = rebar_test_utils:create_random_name("app1_"),
+    Vsn = rebar_test_utils:create_random_vsn(),
+    rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]),
+
+    DepName = rebar_test_utils:create_random_name("dep1_"),
+    PkgName = rebar_test_utils:create_random_name("pkg1_"),
+    mock_git_resource:mock([]),
+    mock_pkg_resource:mock([
+        {pkgdeps, [{{iolist_to_binary(PkgName), iolist_to_binary(Vsn)}, []}]}
+    ]),
+
+    RConfFile = rebar_test_utils:create_config(AppDir, [{deps, [
+        {list_to_atom(DepName), {git, "http://site.com/user/"++DepName++".git", {tag, Vsn}}},
+        {list_to_atom(PkgName), Vsn}
+    ]}]),
+    {ok, RConf} = file:consult(RConfFile),
+
+    %% Build with deps.
+    rebar_test_utils:run_and_check(
+        Config, RConf, ["compile"],
+        {ok, [{app, Name}, {dep, DepName}, {dep, PkgName}]}
+    ),
+
+    %% Build two checkout apps similar to dependencies to be fetched,
+    %% but on a different version
+    Vsn2 = rebar_test_utils:create_random_vsn(),
+    rebar_test_utils:create_app(filename:join([CheckoutsDir,DepName]), DepName, Vsn2, [kernel, stdlib]),
+    rebar_test_utils:create_app(filename:join([CheckoutsDir,PkgName]), PkgName, Vsn2, [kernel, stdlib]),
+
+    %% Rebuild and make sure the checkout apps are in path
+    code:set_path(StartPaths),
+    rebar_test_utils:run_and_check(
+        Config, RConf, ["compile"],
+        {ok, [{app, Name}, {checkout, DepName}, {checkout, PkgName}]}
+    ),
+
+    [DepPath] = [Path || Path <- code:get_path(),
+                         {match, _} <- [re:run(Path, DepName)]],
+    [PkgPath] = [Path || Path <- code:get_path(),
+                         {match, _} <- [re:run(Path, PkgName)]],
+
+    {ok, [DepApp]} = file:consult(filename:join([DepPath, DepName ++ ".app"])),
+    {ok, [PkgApp]} = file:consult(filename:join([PkgPath, PkgName ++ ".app"])),
+
+    {application, _, DepProps} = DepApp,
+    {application, _, PkgProps} = PkgApp,
+
+    ?assertEqual(Vsn2, proplists:get_value(vsn, DepProps)),
+    ?assertEqual(Vsn2, proplists:get_value(vsn, PkgProps)).

test/rebar_test_utils.erl

@@ -25,7 +25,8 @@ init_rebar_state(Config, Name) ->
     ok = ec_file:mkdir_p(CheckoutsDir),
     Verbosity = rebar3:log_level(),
     rebar_log:init(command_line, Verbosity),
-    State = rebar_state:new([{base_dir, filename:join([AppsDir, "_build"])}]),
+    State = rebar_state:new([{base_dir, filename:join([AppsDir, "_build"])}
+                            ,{root_dir, AppsDir}]),
     [{apps, AppsDir}, {checkouts, CheckoutsDir}, {state, State} | Config].
 
 %% @doc Takes common test config, a rebar config ([] if empty), a command to