Merge pull request #179 from ferd/upgrade-pkg

Upgrade pkg

src/rebar_digraph.erl

@@ -71,29 +71,36 @@ restore_graph({Vs, Es}) ->
 cull_deps(Graph, Vertices) ->
     cull_deps(Graph,
               Vertices,
+              1,
+              lists:foldl(fun({Key, _}, Levels) -> dict:store(Key, 0, Levels) end,
+                          dict:new(), Vertices),
               lists:foldl(fun({Key, _}=N, Solution) -> dict:store(Key, N, Solution) end,
                           dict:new(), Vertices),
               []).
 
-cull_deps(_Graph, [], Solution, Discarded) ->
+cull_deps(_Graph, [], _Level, Levels, Solution, Discarded) ->
     {_, Vertices} = lists:unzip(dict:to_list(Solution)),
-    {ok, Vertices, Discarded};
-cull_deps(Graph, Vertices, Solution, Discarded) ->
-    {NV, NS, DS} =
-        lists:foldl(fun(V, {NewVertices, SolutionAcc, DiscardedAcc}) ->
-                        OutNeighbors = digraph:out_neighbours(Graph, V),
-                        lists:foldl(fun({Key, _}=N, {NewVertices1, SolutionAcc1, DiscardedAcc1}) ->
+    LvlVertices = [{App,Vsn,dict:fetch(App,Levels)} || {App,Vsn} <- Vertices],
+    {ok, LvlVertices, Discarded};
+cull_deps(Graph, Vertices, Level, Levels, Solution, Discarded) ->
+    {NV, NS, LS, DS} =
+        lists:foldl(fun(V, {NewVertices, SolutionAcc, LevelsAcc, DiscardedAcc}) ->
+                        OutNeighbors = lists:keysort(1, digraph:out_neighbours(Graph, V)),
+                        lists:foldl(fun({Key, _}=N, {NewVertices1, SolutionAcc1, LevelsAcc1, DiscardedAcc1}) ->
                                             case dict:find(Key, SolutionAcc1) of
                                                 {ok, N} -> % already seen
-                                                    {NewVertices1, SolutionAcc1, DiscardedAcc1};
+                                                    {NewVertices1, SolutionAcc1, LevelsAcc, DiscardedAcc1};
                                                 {ok, _} -> % conflict resolution!
-                                                    {NewVertices1, SolutionAcc1, [N|DiscardedAcc1]};
+                                                    {NewVertices1, SolutionAcc1, LevelsAcc, [N|DiscardedAcc1]};
                                                 error ->
-                                                    {[N | NewVertices1], dict:store(Key, N, SolutionAcc1), DiscardedAcc1}
+                                                    {[N | NewVertices1],
+                                                     dict:store(Key, N, SolutionAcc1),
+                                                     dict:store(Key, Level, LevelsAcc1),
+                                                     DiscardedAcc1}
                                             end
-                                    end, {NewVertices, SolutionAcc, DiscardedAcc}, OutNeighbors)
-                    end, {[], Solution, Discarded}, lists:sort(Vertices)),
-    cull_deps(Graph, NV, NS, DS).
+                                    end, {NewVertices, SolutionAcc, LevelsAcc, DiscardedAcc}, OutNeighbors)
+                    end, {[], Solution, Levels, Discarded}, lists:keysort(1, Vertices)),
+    cull_deps(Graph, NV, Level+1, LS, NS, DS).
 
 subgraph(Graph, Vertices) ->
     digraph_utils:subgraph(Graph, Vertices).

src/rebar_pkg_resource.erl

@@ -14,7 +14,7 @@
 lock(_AppDir, Source) ->
     Source.
 
-needs_update(Dir, {pkg, _Name, Vsn, _Url}) ->
+needs_update(Dir, {pkg, _Name, Vsn}) ->
     [AppInfo] = rebar_app_discover:find_apps([Dir], all),
     case rebar_app_info:original_vsn(AppInfo) =:= ec_cnv:to_list(Vsn) of
         true ->

src/rebar_prv_install_deps.erl

@@ -204,8 +204,9 @@ update_pkg_deps(Profile, Pkgs, Packages, Upgrade, Seen, State) ->
 
 handle_pkg_dep(Profile, Pkg, Packages, Upgrade, DepsDir, Fetched, Seen, State) ->
     AppInfo = package_to_app(DepsDir, Packages, Pkg),
-    {NewSeen, NewState} = maybe_lock(Profile, AppInfo, Seen, State, 0),
-    case maybe_fetch(AppInfo, Upgrade, NewSeen, NewState) of
+    Level = rebar_app_info:dep_level(AppInfo),
+    {NewSeen, NewState} = maybe_lock(Profile, AppInfo, Seen, State, Level),
+    case maybe_fetch(AppInfo, Upgrade, Seen, NewState) of
         true ->
             {[AppInfo | Fetched], NewSeen, NewState};
         false ->
@@ -234,7 +235,7 @@ maybe_lock(Profile, AppInfo, Seen, State, Level) ->
             {Seen, State}
     end.
 
-package_to_app(DepsDir, Packages, {Name, Vsn}) ->
+package_to_app(DepsDir, Packages, {Name, Vsn, Level}) ->
     case dict:find({Name, Vsn}, Packages) of
         error ->
             throw(?PRV_ERROR({missing_package, Name, Vsn}));
@@ -244,7 +245,8 @@ package_to_app(DepsDir, Packages, {Name, Vsn}) ->
             {ok, AppInfo} = rebar_app_info:new(Name, Vsn),
             AppInfo1 = rebar_app_info:deps(AppInfo, PkgDeps),
             AppInfo2 = rebar_app_info:dir(AppInfo1, rebar_dir:deps_dir(DepsDir, Name)),
-            rebar_app_info:source(AppInfo2, {pkg, Name, Vsn})
+            AppInfo3 = rebar_app_info:dep_level(AppInfo2, Level),
+            rebar_app_info:source(AppInfo3, {pkg, Name, Vsn})
     end.
 
 -spec update_src_deps(atom(), non_neg_integer(), list(), list(), list(), rebar_state:t(), boolean(), sets:set(binary()), list()) -> {rebar_state:t(), list(), list(), sets:set(binary())}.
@@ -432,6 +434,8 @@ parse_dep({Name, _Vsn, Source, Opts}, {SrcDepsAcc, PkgDepsAcc}, DepsDir, State)
     ?WARN("Dependency option list ~p in ~p is not supported and will be ignored", [Opts, Name]),
     Dep = new_dep(DepsDir, Name, [], Source, State),
     {[Dep | SrcDepsAcc], PkgDepsAcc};
+parse_dep({_Name, {pkg, Name, Vsn}, Level}, {SrcDepsAcc, PkgDepsAcc}, _, _) when is_integer(Level) ->
+    {SrcDepsAcc, [{Name, Vsn} | PkgDepsAcc]};
 parse_dep({Name, Source, Level}, {SrcDepsAcc, PkgDepsAcc}, DepsDir, State) when is_tuple(Source)
                                                                               , is_integer(Level) ->
     Dep = new_dep(DepsDir, Name, [], Source, State),

src/rebar_prv_update.erl

@@ -47,8 +47,7 @@ do(State) ->
         write_registry(Dict, Graph, State),
         ok
     catch
-        E:C ->
-            io:format("E C ~p ~p~n", [E, C]),
+        _E:_C ->
             throw({error, {?MODULE, package_index_write}})
     end,
 

test/mock_pkg_resource.erl

@@ -47,11 +47,11 @@ mock_lock(_) ->
 %% @doc The config passed to the `mock/2' function can specify which apps
 %% should be updated on a per-name basis: `{update, ["App1", "App3"]}'.
 mock_update(Opts) ->
-    ToUpdate = proplists:get_value(update, Opts, []),
+    ToUpdate = proplists:get_value(upgrade, Opts, []),
     meck:expect(
         ?MOD, needs_update,
         fun(_Dir, {pkg, App, _Vsn}) ->
-            lists:member(App, ToUpdate)
+            lists:member(binary_to_list(App), ToUpdate)
         end).
 
 %% @doc Replicated an unsupported call.

test/rebar_install_deps_SUITE.erl

@@ -39,7 +39,7 @@ init_per_testcase(Case, Config) ->
     mock_warnings(),
     [{expect, Expected},
      {warnings, Warnings}
-    | setup_project(Case, Config, expand_deps(DepsType, Deps))].
+    | setup_project(Case, Config, rebar_test_utils:expand_deps(DepsType, Deps))].
 
 end_per_testcase(_, Config) ->
     meck:unload(),
@@ -110,20 +110,6 @@ deps(circular_skip) ->
      [{"C","2"}],
      {ok, ["B", {"C","1"}, "D"]}}.
 
-expand_deps(_, []) -> [];
-expand_deps(git, [{Name, Deps} | Rest]) ->
-    Dep = {Name, ".*", {git, "https://example.org/user/"++Name++".git", "master"}},
-    [{Dep, expand_deps(git, Deps)} | expand_deps(git, Rest)];
-expand_deps(git, [{Name, Vsn, Deps} | Rest]) ->
-    Dep = {Name, Vsn, {git, "https://example.org/user/"++Name++".git", {tag, Vsn}}},
-    [{Dep, expand_deps(git, Deps)} | expand_deps(git, Rest)];
-expand_deps(pkg, [{Name, Deps} | Rest]) ->
-    Dep = {pkg, Name, "0.0.0", "https://example.org/user/"++Name++".tar.gz"},
-    [{Dep, expand_deps(pkg, Deps)} | expand_deps(pkg, Rest)];
-expand_deps(pkg, [{Name, Vsn, Deps} | Rest]) ->
-    Dep = {pkg, Name, Vsn, "https://example.org/user/"++Name++".tar.gz"},
-    [{Dep, expand_deps(pkg, Deps)} | expand_deps(pkg, Rest)].
-
 setup_project(Case, Config0, Deps) ->
     DepsType = ?config(deps_type, Config0),
     Config = rebar_test_utils:init_rebar_state(
@@ -132,50 +118,16 @@ setup_project(Case, Config0, Deps) ->
     ),
     AppDir = ?config(apps, Config),
     rebar_test_utils:create_app(AppDir, "A", "0.0.0", [kernel, stdlib]),
-    TopDeps = top_level_deps(Deps),
+    TopDeps = rebar_test_utils:top_level_deps(Deps),
     RebarConf = rebar_test_utils:create_config(AppDir, [{deps, TopDeps}]),
     case DepsType of
         git ->
-            mock_git_resource:mock([{deps, flat_deps(Deps)}]);
+            mock_git_resource:mock([{deps, rebar_test_utils:flat_deps(Deps)}]);
         pkg ->
-            mock_pkg_resource:mock([{pkgdeps, flat_pkgdeps(Deps)}])
+            mock_pkg_resource:mock([{pkgdeps, rebar_test_utils:flat_pkgdeps(Deps)}])
     end,
     [{rebarconfig, RebarConf} | Config].
 
-
-flat_deps([]) -> [];
-flat_deps([{{Name,_Vsn,Ref}, Deps} | Rest]) ->
-    [{{Name,vsn_from_ref(Ref)}, top_level_deps(Deps)}]
-    ++
-    flat_deps(Deps)
-    ++
-    flat_deps(Rest).
-
-vsn_from_ref({git, _, {_, Vsn}}) -> Vsn;
-vsn_from_ref({git, _, Vsn}) -> Vsn.
-
-flat_pkgdeps([]) -> [];
-flat_pkgdeps([{{pkg, Name, Vsn, _Url}, Deps} | Rest]) ->
-    [{{iolist_to_binary(Name),iolist_to_binary(Vsn)}, top_level_deps(Deps)}]
-    ++
-    flat_pkgdeps(Deps)
-    ++
-    flat_pkgdeps(Rest).
-
-top_level_deps([]) -> [];
-top_level_deps([{{Name, Vsn, Ref}, _} | Deps]) ->
-    [{list_to_atom(Name), Vsn, Ref} | top_level_deps(Deps)];
-top_level_deps([{{pkg, Name, Vsn, _URL}, _} | Deps]) ->
-    [{list_to_atom(Name), Vsn} | top_level_deps(Deps)].
-
-app_vsn([]) -> [];
-app_vsn([{Source, Deps} | Rest]) ->
-    {Name, Vsn} = case Source of
-        {N,V,_Ref} -> {N,V};
-        {pkg, N, V, _} -> {N,V}
-    end,
-    [{Name, Vsn}] ++ app_vsn(Deps) ++ app_vsn(Rest).
-
 mock_warnings() ->
     %% just let it do its thing, we check warnings through
     %% the call log.

test/rebar_test_utils.erl

@@ -2,7 +2,7 @@
 -include_lib("common_test/include/ct.hrl").
 -include_lib("eunit/include/eunit.hrl").
 -export([init_rebar_state/1, init_rebar_state/2, run_and_check/4]).
--export([expand_deps/2, flat_deps/1, top_level_deps/1]).
+-export([expand_deps/2, flat_deps/1, flat_pkgdeps/1, top_level_deps/1]).
 -export([create_app/4, create_empty_app/4, create_config/2]).
 -export([create_random_name/1, create_random_vsn/0]).
 
@@ -120,6 +120,15 @@ flat_deps([{{Name,_Vsn,Ref}, Deps} | Rest]) ->
     ++
     flat_deps(Rest).
 
+flat_pkgdeps([]) -> [];
+flat_pkgdeps([{{pkg, Name, Vsn}, Deps} | Rest]) ->
+    [{{iolist_to_binary(Name),iolist_to_binary(Vsn)}, top_level_deps(Deps)}]
+    ++
+    flat_pkgdeps(Deps)
+    ++
+    flat_pkgdeps(Rest).
+
+
 vsn_from_ref({git, _, {_, Vsn}}) -> Vsn;
 vsn_from_ref({git, _, Vsn}) -> Vsn.
 
@@ -196,6 +205,9 @@ check_results(AppDir, Expected) ->
                 case lists:keyfind(iolist_to_binary(Name), 1, Locks) of
                     false ->
                         error({lock_not_found, Name});
+                    {_LockName, {pkg, _, LockVsn}, _} ->
+                        ?assertEqual(iolist_to_binary(Vsn),
+                                     iolist_to_binary(LockVsn));
                     {_LockName, {_, _, {ref, LockVsn}}, _} ->
                         ?assertEqual(iolist_to_binary(Vsn),
                                      iolist_to_binary(LockVsn))

test/rebar_upgrade_SUITE.erl

@@ -3,7 +3,7 @@
 -include_lib("eunit/include/eunit.hrl").
 -compile(export_all).
 
-all() -> [{group, git}].%, {group, pkg}].
+all() -> [{group, git}, {group, pkg}].
 
 groups() ->
     [{all, [], [top_a, top_b, top_c, top_d1, top_d2, top_e,
@@ -34,11 +34,11 @@ end_per_group(_, Config) ->
 init_per_testcase(Case, Config) ->
     DepsType = ?config(deps_type, Config),
     {Deps, UpDeps, ToUp, Expectations} = upgrades(Case),
-    Expanded = expand_deps(DepsType, Deps),
-    UpExpanded = expand_deps(DepsType, UpDeps),
+    Expanded = rebar_test_utils:expand_deps(DepsType, Deps),
+    UpExpanded = rebar_test_utils:expand_deps(DepsType, UpDeps),
     [{expected, normalize_unlocks(Expectations)},
      {mock, fun() -> mock_deps(DepsType, Expanded, []) end},
-     {mock_update, fun() -> mock_deps(DepsType, UpExpanded, ToUp) end}
+     {mock_update, fun() -> mock_deps(DepsType, Expanded, UpExpanded, ToUp) end}
      | setup_project(Case, Config, Expanded, UpExpanded)].
 
 end_per_testcase(_, Config) ->
@@ -53,10 +53,10 @@ setup_project(Case, Config0, Deps, UpDeps) ->
     ),
     AppDir = ?config(apps, Config),
     rebar_test_utils:create_app(AppDir, "Root", "0.0.0", [kernel, stdlib]),
-    TopDeps = top_level_deps(Deps),
+    TopDeps = rebar_test_utils:top_level_deps(Deps),
     RebarConf = rebar_test_utils:create_config(AppDir, [{deps, TopDeps}]),
     [{rebarconfig, RebarConf},
-     {next_top_deps, top_level_deps(UpDeps)} | Config].
+     {next_top_deps, rebar_test_utils:top_level_deps(UpDeps)} | Config].
 
 
 upgrades(top_a) ->
@@ -206,7 +206,7 @@ upgrades(triplet_b) ->
                   {"G",[]}]},
       {"C", "0", [{"H","3",[]},
                   {"I",[]}]}],
-     [{"A", "1", [{"D",[]},
+     [{"A", "2", [{"D",[]},
                   {"E","2",[]}]},
       {"B", "1", [{"F","1",[]},
                   {"G",[]}]},
@@ -223,7 +223,7 @@ upgrades(triplet_c) ->
                   {"G",[]}]},
       {"C", "0", [{"H","3",[]},
                   {"I",[]}]}],
-     [{"A", "1", [{"D",[]},
+     [{"A", "2", [{"D",[]},
                   {"E","2",[]}]},
       {"B", "1", [{"F","1",[]},
                   {"G",[]}]},
@@ -245,7 +245,7 @@ upgrades(tree_a) ->
                   {"E",[{"I","1",[]}]}]},
       {"B", "1", [{"F",[]},
                   {"G",[]}]},
-      {"C", "1", [{"H",[]}]}
+      {"C", "2", [{"H",[]}]}
      ],
      ["C"],
      {"A", [{"A","1"}, "D", "J", "E",
@@ -263,7 +263,7 @@ upgrades(tree_b) ->
                   {"E",[{"I","1",[]}]}]},
       {"B", "1", [{"F",[]},
                   {"G",[]}]},
-      {"C", "1", [{"H",[]}]}
+      {"C", "2", [{"H",[]}]}
      ],
      ["C"],
      {"B", [{"A","1"}, "D", "J", "E",
@@ -356,51 +356,21 @@ upgrades(delete_d) ->
 %% TODO: add a test that verifies that unlocking files and then
 %% running the upgrade code is enough to properly upgrade things.
 
-top_level_deps([]) -> [];
-top_level_deps([{{pkg, Name, Vsn}, _} | Deps]) ->
-    [{list_to_atom(Name), Vsn} | top_level_deps(Deps)];
-top_level_deps([{{Name, Vsn, Ref}, _} | Deps]) ->
-    [{list_to_atom(Name), Vsn, Ref} | top_level_deps(Deps)].
-
 mock_deps(git, Deps, Upgrades) ->
     catch mock_git_resource:unmock(),
-    mock_git_resource:mock([{deps, flat_deps(Deps)}, {upgrade, Upgrades}]);
+    mock_git_resource:mock([{deps, rebar_test_utils:flat_deps(Deps)}, {upgrade, Upgrades}]);
 mock_deps(pkg, Deps, Upgrades) ->
     catch mock_pkg_resource:unmock(),
-    mock_pkg_resource:mock([{pkgdeps, flat_pkgdeps(Deps)}, {upgrade, Upgrades}]).
+    mock_pkg_resource:mock([{pkgdeps, rebar_test_utils:flat_pkgdeps(Deps)}, {upgrade, Upgrades}]).
 
-flat_deps([]) -> [];
-flat_deps([{{Name,_Vsn,Ref}, Deps} | Rest]) ->
-    [{{Name,vsn_from_ref(Ref)}, top_level_deps(Deps)}]
-    ++
-    flat_deps(Deps)
-    ++
-    flat_deps(Rest).
-
-vsn_from_ref({git, _, {_, Vsn}}) -> Vsn;
-vsn_from_ref({git, _, Vsn}) -> Vsn.
-
-flat_pkgdeps([]) -> [];
-flat_pkgdeps([{{pkg, Name, Vsn}, Deps} | Rest]) ->
-    [{{iolist_to_binary(Name),iolist_to_binary(Vsn)}, top_level_deps(Deps)}]
-    ++
-    flat_pkgdeps(Deps)
-    ++
-    flat_pkgdeps(Rest).
-
-expand_deps(_, []) -> [];
-expand_deps(git, [{Name, Deps} | Rest]) ->
-    Dep = {Name, ".*", {git, "https://example.org/user/"++Name++".git", "master"}},
-    [{Dep, expand_deps(git, Deps)} | expand_deps(git, Rest)];
-expand_deps(git, [{Name, Vsn, Deps} | Rest]) ->
-    Dep = {Name, Vsn, {git, "https://example.org/user/"++Name++".git", {tag, Vsn}}},
-    [{Dep, expand_deps(git, Deps)} | expand_deps(git, Rest)];
-expand_deps(pkg, [{Name, Deps} | Rest]) ->
-    Dep = {pkg, Name, "0.0.0"},
-    [{Dep, expand_deps(pkg, Deps)} | expand_deps(pkg, Rest)];
-expand_deps(pkg, [{Name, Vsn, Deps} | Rest]) ->
-    Dep = {pkg, Name, Vsn},
-    [{Dep, expand_deps(pkg, Deps)} | expand_deps(pkg, Rest)].
+mock_deps(git, _OldDeps, Deps, Upgrades) ->
+    catch mock_git_resource:unmock(),
+    mock_git_resource:mock([{deps, rebar_test_utils:flat_deps(Deps)}, {upgrade, Upgrades}]);
+mock_deps(pkg, OldDeps, Deps, Upgrades) ->
+    Merged = Deps ++ [Dep || Dep <- OldDeps,
+                             not lists:keymember(element(1, Dep), 1, Deps)],
+    catch mock_pkg_resource:unmock(),
+    mock_pkg_resource:mock([{pkgdeps, rebar_test_utils:flat_pkgdeps(Merged)}, {upgrade, Upgrades}]).
 
 normalize_unlocks({App, Locks}) ->
     {iolist_to_binary(App),