Merge pull request #732 from tsloughter/level_wins

this patch treats pkg and src deps as equals, so level decides winner

src/rebar.hrl

@@ -23,8 +23,14 @@
 -define(DEFAULT_RELEASE_DIR, "rel").
 -define(DEFAULT_CONFIG_FILE, "rebar.config").
 -define(DEFAULT_CDN, "https://s3.amazonaws.com/s3.hex.pm/tarballs").
+-define(DEFAULT_HEX_REGISTRY, "https://s3.amazonaws.com/s3.hex.pm/registry.ets.gz").
 -define(LOCK_FILE, "rebar.lock").
 
+-define(PACKAGE_INDEX_VERSION, 3).
+-define(PACKAGE_TABLE, package_index).
+-define(INDEX_FILE, "packages.idx").
+-define(REGISTRY_FILE, "registry").
+
 -ifdef(namespaced_types).
 -type rebar_dict() :: dict:dict().
 -else.

src/rebar3.erl

@@ -105,6 +105,7 @@ run_aux(State, RawArgs) ->
     {ok, Providers} = application:get_env(rebar, providers),
     %% Providers can modify profiles stored in opts, so set default after initializing providers
     State4 = rebar_state:create_logic_providers(Providers, State3),
+    rebar_packages:packages(State4),
     State5 = rebar_plugins:project_apps_install(State4),
     State6 = rebar_state:default(State5, rebar_state:opts(State5)),
 
@@ -280,9 +281,15 @@ state_from_global_config(Config, GlobalConfigFile) ->
     %% We don't want to worry about global plugin install state effecting later
     %% usage. So we throw away the global profile state used for plugin install.
     GlobalConfigThrowAway = rebar_state:current_profiles(GlobalConfig, [global]),
-    GlobalState = rebar_plugins:handle_plugins(global,
-                                               rebar_state:get(GlobalConfigThrowAway, plugins, []),
-                                               GlobalConfigThrowAway),
+    GlobalState = case rebar_state:get(GlobalConfigThrowAway, plugins, []) of
+                      [] ->
+                          GlobalConfigThrowAway;
+                      GlobalPluginsToInstall ->
+                          rebar_packages:packages(GlobalConfigThrowAway),
+                          rebar_plugins:handle_plugins(global,
+                                                       GlobalPluginsToInstall,
+                                                       GlobalConfigThrowAway)
+                  end,
     GlobalPlugins = rebar_state:providers(GlobalState),
     GlobalConfig2 = rebar_state:set(GlobalConfig, plugins, []),
     GlobalConfig3 = rebar_state:set(GlobalConfig2, {plugins, global}, rebar_state:get(GlobalConfigThrowAway, plugins, [])),

src/rebar_app_discover.erl

@@ -96,7 +96,7 @@ merge_deps(AppInfo, State) ->
     {AppInfo1, State1}.
 
 handle_profile(Profile, Name, AppState, State) ->
-    {TopSrc, TopPkg} = rebar_state:get(State, {parsed_deps, Profile}, {[], []}),
+    TopParsedDeps = rebar_state:get(State, {parsed_deps, Profile}, {[], []}),
     TopLevelProfileDeps = rebar_state:get(State, {deps, Profile}, []),
     AppProfileDeps = rebar_state:get(AppState, {deps, Profile}, []),
     AppProfileDeps2 = rebar_utils:tup_dedup(rebar_utils:tup_sort(AppProfileDeps)),
@@ -108,18 +108,18 @@ handle_profile(Profile, Name, AppState, State) ->
     %% Only deps not also specified in the top level config need
     %% to be included in the parsed deps
     NewDeps = ProfileDeps2 -- TopLevelProfileDeps,
-    {ParsedSrc, ParsedPkg} = parse_profile_deps(Profile, Name, NewDeps, AppState, State1),
-    rebar_state:set(State1, {parsed_deps, Profile}, {TopSrc++ParsedSrc, TopPkg++ParsedPkg}).
+    ParsedDeps = parse_profile_deps(Profile, Name, NewDeps, AppState, State1),
+    rebar_state:set(State1, {parsed_deps, Profile}, TopParsedDeps++ParsedDeps).
 
 parse_profile_deps(Profile, Name, Deps, AppState, State) ->
     DepsDir = rebar_prv_install_deps:profile_dep_dir(State, Profile),
     Locks = rebar_state:get(State, {locks, Profile}, []),
-    rebar_prv_install_deps:parse_deps(Name
-                                     ,DepsDir
-                                     ,Deps
-                                     ,AppState
-                                     ,Locks
-                                     ,1).
+    rebar_app_utils:parse_deps(Name
+                              ,DepsDir
+                              ,Deps
+                              ,AppState
+                              ,Locks
+                              ,1).
 
 project_app_config(AppInfo, State) ->
     C = rebar_config:consult(rebar_app_info:dir(AppInfo)),

src/rebar_app_info.erl

@@ -34,6 +34,8 @@
          dir/2,
          out_dir/1,
          out_dir/2,
+         resource_type/1,
+         resource_type/2,
          source/1,
          source/2,
          state/1,
@@ -64,6 +66,7 @@
                      dep_level=0        :: integer(),
                      dir                :: file:name(),
                      out_dir            :: file:name(),
+                     resource_type      :: pkg | src,
                      source             :: string() | tuple() | undefined,
                      state              :: rebar_state:t() | undefined,
                      is_lock=false      :: boolean(),
@@ -274,6 +277,14 @@ out_dir(AppInfo=#app_info_t{}, OutDir) ->
 ebin_dir(#app_info_t{out_dir=OutDir}) ->
     ec_cnv:to_list(filename:join(OutDir, "ebin")).
 
+-spec resource_type(t(), pkg | src) -> t().
+resource_type(AppInfo=#app_info_t{}, Type) ->
+    AppInfo#app_info_t{resource_type=Type}.
+
+-spec resource_type(t()) -> pkg | src.
+resource_type(#app_info_t{resource_type=ResourceType}) ->
+    ResourceType.
+
 -spec source(t(), string() | tuple()) -> t().
 source(AppInfo=#app_info_t{}, Source) ->
     AppInfo#app_info_t{source=Source}.
@@ -316,7 +327,7 @@ is_checkout(#app_info_t{is_checkout=IsCheckout}) ->
 
 -spec valid(t()) -> boolean().
 valid(AppInfo=#app_info_t{valid=undefined, state=State}) ->
-    case rebar_app_utils:validate_application_info(AppInfo)
+    case rebar_app_utils:validate_application_info(AppInfo) =:= true
         andalso rebar_state:has_all_artifacts(State) =:= true of
         true ->
             true;

src/rebar_app_utils.erl

@@ -32,6 +32,9 @@
          app_src_to_app/2,
          validate_application_info/1,
          validate_application_info/2,
+         parse_deps/5,
+         parse_deps/6,
+         dep_to_app/7,
          format_error/1]).
 
 -include("rebar.hrl").
@@ -87,6 +90,109 @@ validate_application_info(AppInfo, AppDetail) ->
             end
     end.
 
+-spec parse_deps(binary(), list(), rebar_state:t(), list(), integer()) -> {[rebar_app_info:t()], [tuple()]}.
+parse_deps(DepsDir, Deps, State, Locks, Level) ->
+    parse_deps(root, DepsDir, Deps, State, Locks, Level).
+
+parse_deps(Parent, DepsDir, Deps, State, Locks, Level) ->
+    [parse_dep(Dep, Parent, DepsDir, State, Locks, Level) || Dep <- Deps].
+
+parse_dep(Dep, Parent, DepsDir, State, Locks, Level) ->
+    Name = case Dep of
+               Dep when is_tuple(Dep) ->
+                   element(1, Dep);
+               Dep ->
+                   Dep
+           end,
+    case lists:keyfind(ec_cnv:to_binary(Name), 1, Locks) of
+        false ->
+            parse_dep(Parent, Dep, DepsDir, false, State);
+        LockedDep ->
+            LockedLevel = element(3, LockedDep),
+            case LockedLevel > Level of
+                true ->
+                    parse_dep(Parent, Dep, DepsDir, false, State);
+                false ->
+                    parse_dep(Parent, LockedDep, DepsDir, true, State)
+            end
+    end.
+
+parse_dep(Parent, {Name, Vsn}, DepsDir, IsLock, State) when is_list(Vsn); is_binary(Vsn) ->
+    %% Versioned Package dependency
+    CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)),
+    case rebar_app_info:discover(CheckoutsDir) of
+        {ok, _App} ->
+            dep_to_app(root, DepsDir, Name, [], [], IsLock, State);
+        not_found ->
+            {PkgName, PkgVsn} = parse_goal(ec_cnv:to_binary(Name)
+                                          ,ec_cnv:to_binary(Vsn)),
+            Source = {pkg, PkgName, PkgVsn},
+            rebar_app_info:resource_type(dep_to_app(Parent, DepsDir, PkgName, PkgVsn, Source, IsLock, State), pkg)
+    end;
+parse_dep(Parent, Name, DepsDir, IsLock, State) when is_atom(Name); is_binary(Name) ->
+    %% Unversioned package dependency
+    {PkgName, PkgVsn} = get_package(ec_cnv:to_binary(Name), State),
+    CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)),
+    case rebar_app_info:discover(CheckoutsDir) of
+        {ok, _App} ->
+            dep_to_app(root, DepsDir, Name, [], [], IsLock, State);
+        not_found ->
+            Source = {pkg, PkgName, PkgVsn},
+            rebar_app_info:resource_type(dep_to_app(Parent, DepsDir, PkgName, PkgVsn, Source, IsLock, State), pkg)
+    end;
+parse_dep(Parent, {Name, Source}, DepsDir, IsLock, State) when is_tuple(Source) ->
+    dep_to_app(Parent, DepsDir, Name, [], Source, IsLock, State);
+parse_dep(Parent, {Name, _Vsn, Source}, DepsDir, IsLock, State) when is_tuple(Source) ->
+    dep_to_app(Parent, DepsDir, Name, [], Source, IsLock, State);
+parse_dep(Parent, {Name, _Vsn, Source, Opts}, DepsDir, IsLock, State) when is_tuple(Source) ->
+    ?WARN("Dependency option list ~p in ~p is not supported and will be ignored", [Opts, Name]),
+    dep_to_app(Parent, DepsDir, Name, [], Source, IsLock, State);
+parse_dep(Parent, {_Name, {pkg, Name, Vsn}, Level}, DepsDir, IsLock, State) when is_integer(Level) ->
+    CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)),
+    case rebar_app_info:discover(CheckoutsDir) of
+        {ok, _App} ->
+            dep_to_app(root, DepsDir, Name, [], [], IsLock, State);
+        not_found ->
+            Source = {pkg, Name, Vsn},
+            rebar_app_info:resource_type(dep_to_app(Parent, DepsDir, Name, Vsn, Source, IsLock, State), pkg)
+    end;
+parse_dep(Parent, {Name, Source, Level}, DepsDir, IsLock, State) when is_tuple(Source)
+                                                                    , is_integer(Level) ->
+    dep_to_app(Parent, DepsDir, Name, [], Source, IsLock, State);
+parse_dep(_, Dep, _, _, _) ->
+    throw(?PRV_ERROR({parse_dep, Dep})).
+
+dep_to_app(Parent, DepsDir, Name, Vsn, Source, IsLock, State) ->
+    CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)),
+    BaseDir = rebar_state:get(State, base_dir, []),
+    {ok, Dep} = case rebar_app_info:discover(CheckoutsDir) of
+                    {ok, App} ->
+                        {ok, rebar_app_info:is_checkout(App, true)};
+                    not_found ->
+                        Dir = ec_cnv:to_list(filename:join(DepsDir, Name)),
+                        case rebar_app_info:discover(Dir) of
+                            {ok, App} ->
+                                {ok, App};
+                            not_found ->
+                                rebar_app_info:new(Name, Vsn,
+                                                   ec_cnv:to_list(filename:join(DepsDir, Name)))
+                        end
+                end,
+    C = rebar_config:consult(rebar_app_info:dir(Dep)),
+    S = rebar_state:new(rebar_state:new(), C, rebar_app_info:dir(Dep)),
+    Overrides = rebar_state:get(State, overrides, []),
+    ParentOverrides = rebar_state:overrides(State),
+    S1 = rebar_state:set(rebar_state:overrides(S, ParentOverrides++Overrides), base_dir, BaseDir),
+    Dep1 = rebar_app_info:state(Dep, S1),
+    AppInfo = rebar_app_info:is_lock(rebar_app_info:source(Dep1, Source), IsLock),
+    ResourceType = case Source of
+                       {pkg, _, _} ->
+                           pkg;
+                       _ ->
+                           src
+                   end,
+    rebar_app_info:resource_type(rebar_app_info:parent(AppInfo, Parent), ResourceType).
+
 format_error(Error) ->
     io_lib:format("~p", [Error]).
 
@@ -94,11 +200,25 @@ format_error(Error) ->
 %% Internal functions
 %% ===================================================================
 
+-spec parse_goal(binary(), binary()) -> {binary(), binary()} | {binary(), binary(), binary()}.
+parse_goal(Name, Constraint) ->
+    case re:run(Constraint, "([^\\d]*)(\\d.*)", [{capture, [1,2], binary}]) of
+        {match, [<<>>, Vsn]} ->
+            {Name, Vsn};
+        {match, [Op, Vsn]} ->
+            {Name, Vsn, binary_to_atom(Op, utf8)};
+        nomatch ->
+            throw(?PRV_ERROR({bad_constraint, Name, Constraint}))
+    end.
+
+get_package(Dep, State) ->
+    {ok, HighestDepVsn} = rebar_packages:find_highest_matching(Dep, "0", ?PACKAGE_TABLE, State),
+    {Dep, HighestDepVsn}.
+
 -spec has_all_beams(file:filename_all(), [module()]) ->
     true | ?PRV_ERROR({missing_module, module()}).
 has_all_beams(EbinDir, [Module | ModuleList]) ->
-    BeamFile = filename:join([EbinDir,
-                              ec_cnv:to_list(Module) ++ ".beam"]),
+    BeamFile = filename:join([EbinDir, ec_cnv:to_list(Module) ++ ".beam"]),
     case filelib:is_file(BeamFile) of
         true ->
             has_all_beams(EbinDir, ModuleList);

src/rebar_digraph.erl

@@ -2,8 +2,6 @@
 
 -export([compile_order/1
         ,restore_graph/1
-        ,cull_deps/2
-        ,cull_deps/3
         ,subgraph/2
         ,format_error/1]).
 
@@ -69,17 +67,6 @@ restore_graph({Vs, Es}) ->
                   end, Es),
     Graph.
 
-%% Pick packages to fullfill dependencies
-%% The first dep while traversing the graph is chosen and any conflicting
-%% dep encountered later on is ignored.
-cull_deps(Graph, Vertices) ->
-    cull_deps(Graph, Vertices, sets:new()).
-
-cull_deps(Graph, Vertices, Seen) ->
-    Vertices1 = lists:keysort(2, Vertices),
-    {Solution, Levels, Discarded} = {dict:new(), dict:new(), sets:new()},
-    cull_deps(Graph, Vertices1, Levels, Solution, Seen, Discarded).
-
 format_error(no_solution) ->
     io_lib:format("No solution for packages found.", []).
 
@@ -87,103 +74,6 @@ format_error(no_solution) ->
 %% Internal Functions
 %%====================================================================
 
-cull_deps(_Graph, [], Levels, Solution, _, Discarded) ->
-    {_, Vertices} = lists:unzip(dict:to_list(Solution)),
-    LvlVertices = [{Profile, {Parent, App, Vsn, dict:fetch(App, Levels)}}
-                  || {Profile, {Parent,App,Vsn}} <- Vertices],
-    {ok, LvlVertices,  sets:to_list(Discarded)};
-cull_deps(Graph, [{Profile, Level, Vs} | Vertices], Levels, Solution, Seen, Discarded) ->
-    {NV, NS, LS, DS} =
-        lists:foldl(fun({Parent, Name, Vsn}, {Acc, SolutionAcc, LevelsAcc, DiscardedAcc}) ->
-                            {SolutionAcc1, LevelsAcc1, DiscardedAcc1} =
-                                maybe_add_to_solution(Profile, Level, Name, {Name, Vsn}, Parent
-                                                     ,SolutionAcc
-                                                     ,LevelsAcc, Seen, DiscardedAcc),
-                            OutNeighbors = digraph:out_neighbours(Graph, {Name,Vsn}),
-                            {NewVertices, DiscardedAcc2} = handle_neighbors(Profile, Level, Name
-                                                          ,OutNeighbors, Acc, SolutionAcc1
-                                                                           ,Seen, DiscardedAcc1),
-                            {NewVertices, SolutionAcc1, LevelsAcc1, DiscardedAcc2}
-                    end, {[], Solution, Levels, Discarded}, Vs),
-    NewVertices = combine_profile_levels(Vertices, NV),
-    cull_deps(Graph, NewVertices, LS, NS, Seen, DS).
-
-%% Combine lists of deps that have the same profile and level
-combine_profile_levels(Vertices, NewVertices) ->
-    V = lists:foldl(fun({Profile, Level, Vs}, Acc) ->
-                            case ec_lists:find(fun({P, L, _}) ->
-                                                       P =:= Profile andalso L =:= Level
-                                               end, Acc) of
-                            {ok, {_, _, OldVs}=Old} ->
-                                lists:delete(Old, Acc)++[{Profile, Level, lists:keysort(1, OldVs++Vs)}];
-                            error ->
-                                Acc++[{Profile, Level, Vs}]
-                        end
-                end, Vertices, NewVertices),
-    lists:keysort(2, V).
-
-%% For each outgoing edge of a dep check if it should be added to the solution
-%% and add it to the list of vertices to do the same for
-handle_neighbors(Profile, Level, Parent, OutNeighbors, Vertices
-                ,Solution, Seen, Discarded) ->
-    case lists:foldl(fun({Name, Vsn}=Value, {NewVertices, Discarded1}) ->
-                             case dict:find(Name, Solution) of
-                                 {ok, {Profile, {Parent, Name, Vsn}}} -> % already seen
-                                     {NewVertices,
-                                      Discarded1};
-                                 {ok, _} -> % conflict resolution!
-                                     %% Warn on different version
-                                     {NewVertices,
-                                      sets:add_element(Value, Discarded1)};
-                                 error ->
-                                     %% We check Seen separately because we don't care
-                                     %% to warn if the exact same version of a package
-                                     %% was already part of the solution but we do
-                                     %% if it was simply seen in source deps
-                                     case sets:is_element(Name, Seen) of
-                                         true ->
-                                             {NewVertices,
-                                              sets:add_element(Value, Discarded1)};
-                                         false ->
-                                             {[{Parent, Name, Vsn} | NewVertices],
-                                              Discarded1}
-                                     end
-                             end
-                    end, {[], Discarded}, OutNeighbors) of
-            {[], DiscardedAcc2} ->
-                {Vertices, DiscardedAcc2};
-            {NewVertices1, DiscardedAcc2} ->
-                {Vertices++[{Profile, Level+1, NewVertices1}] ,DiscardedAcc2}
-        end.
-
-maybe_add_to_solution(Profile, Level, Key, {Name, Vsn}=Value, Parent
-                     ,Solution, Levels, Seen, Discarded) ->
-    case dict:find(Key, Solution) of
-        {ok, {Profile, {Parent, Name, Vsn}}} -> % already seen
-            {Solution,
-             Levels,
-             Discarded};
-        {ok, _} -> % conflict resolution!
-            %% Warn on different version
-            {Solution,
-             Levels,
-             sets:add_element(Value, Discarded)};
-        error ->
-            %% We check Seen separately because we don't care to warn if the exact
-            %% same version of a package was already part of the solution but we do
-            %% if it was simply seen in source deps
-            case sets:is_element(Name, Seen) of
-                true ->
-                    {Solution,
-                     Levels,
-                     sets:add_element(Value, Discarded)};
-                false ->
-                    {dict:store(Key, {Profile, {Parent, Name, Vsn}}, Solution),
-                     dict:store(Key, Level, Levels),
-                     Discarded}
-            end
-    end.
-
 subgraph(Graph, Vertices) ->
     digraph_utils:subgraph(Graph, Vertices).
 

src/rebar_packages.erl

@@ -1,59 +1,60 @@
 -module(rebar_packages).
 
 -export([packages/1
-        ,registry/1
+        ,close_packages/0
+        ,load_and_verify_version/1
+        ,deps/3
         ,package_dir/1
-        ,check_registry/3
         ,registry_checksum/2
-        ,find_highest_matching/3]).
+        ,find_highest_matching/4
+        ,format_error/1]).
 
 -export_type([package/0]).
 
 -include("rebar.hrl").
+-include_lib("providers/include/providers.hrl").
 
 -type pkg_name() :: binary() | atom().
 -type vsn() :: binary().
 -type package() :: pkg_name() | {pkg_name(), vsn()}.
 
--spec packages(rebar_state:t()) -> {rebar_dict(), rebar_digraph()}.
-%% DON'T USE IT! Use rebar_state:packages(State) instead.
+-spec packages(rebar_state:t()) -> ets:tid().
 packages(State) ->
-    RegistryDir = package_dir(State),
-    DictFile = filename:join(RegistryDir, "dict"),
-    Edges = filename:join(RegistryDir, "edges"),
-    Vertices = filename:join(RegistryDir, "vertices"),
-    Neighbors = filename:join(RegistryDir, "neighbors"),
-
-    case lists:all(fun(X) -> filelib:is_file(X) end, [DictFile, Edges, Vertices, Neighbors]) of
+    catch ets:delete(?PACKAGE_TABLE),
+    case load_and_verify_version(State) of
         true ->
-            try
-                {ok, DictBinary} = file:read_file(DictFile),
-                Dict = binary_to_term(DictBinary),
-                {ok, EdgesTab} = ets:file2tab(Edges),
-                {ok, VerticesTab} = ets:file2tab(Vertices),
-                {ok, NeighborsTab} = ets:file2tab(Neighbors),
-                {Dict, {digraph, EdgesTab, VerticesTab, NeighborsTab, true}}
-            catch
-                _:_ ->
-                    ?ERROR("Bad packages index, try to fix with `rebar3 update`", []),
-                    {dict:new(), digraph:new()}
-            end;
+            ok;
         false ->
+            ?DEBUG("Error loading package index.", []),
             ?ERROR("Bad packages index, try to fix with `rebar3 update`", []),
-            {dict:new(), digraph:new()}
+            ets:new(?PACKAGE_TABLE, [named_table, public])
     end.
 
--spec registry(rebar_state:t()) -> {ok, ets:tid()} | {error, any()}.
-%% DON'T USE IT! Use rebar_state:registry(State) instead.
-registry(State) ->
+close_packages() ->
+    catch ets:delete(?PACKAGE_TABLE).
+
+load_and_verify_version(State) ->
     RegistryDir = package_dir(State),
-    HexFile = filename:join(RegistryDir, "registry"),
-    case ets:file2tab(HexFile) of
-        {ok, T} ->
-            {ok, T};
-        {error, Reason} ->
-            ?DEBUG("Error loading registry: ~p", [Reason]),
-            error
+    case ets:file2tab(filename:join(RegistryDir, ?INDEX_FILE)) of
+        {ok, _} ->
+            case ets:lookup_element(?PACKAGE_TABLE, package_index_version, 2) of
+                ?PACKAGE_INDEX_VERSION ->
+                    true;
+                _ ->
+                    (catch ets:delete(?PACKAGE_TABLE)),
+                    rebar_prv_update:hex_to_index(State)
+            end;
+        _ ->
+            rebar_prv_update:hex_to_index(State)
+    end.
+
+deps(Name, Vsn, State) ->
+    try
+        verify_table(State),
+        ets:lookup_element(?PACKAGE_TABLE, {ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)}, 2)
+    catch
+        _:_ ->
+            throw(?PRV_ERROR({missing_package, ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)}))
     end.
 
 package_dir(State) ->
@@ -66,27 +67,13 @@ package_dir(State) ->
     ok = filelib:ensure_dir(filename:join(PackageDir, "placeholder")),
     PackageDir.
 
-
-check_registry(Pkg, Vsn, State) ->
-    case rebar_state:registry(State) of
-        {ok, T} ->
-            case ets:lookup(T, Pkg) of
-                [{Pkg, [Vsns]}] ->
-                    lists:member(Vsn, Vsns);
-                _ ->
-                    false
-            end;
-        error ->
-            false
-    end.
-
 registry_checksum({pkg, Name, Vsn}, State) ->
-    {ok
, Registry} = rebar_state:registry(State),
-    case ets:lookup(Registry, {Name, Vsn}) of
-        [{{_, _}, [_, Checksum | _]}] ->
-            Checksum;
-        [] ->
-            none
+    try
+        verify_table(State),
+        ets:lookup_element(?PACKAGE_TABLE, {Name, Vsn}, 3)
+    catch
+        _:_ ->
+            throw(?PRV_ERROR({missing_package, ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)}))
     end.
 
 %% Hex supports use of ~> to specify the version required for a dependency.
@@ -104,28 +91,45 @@ registry_checksum({pkg, Name, Vsn}, State) ->
 %% `~> 2.1.3-dev` | `>= 2.1.3-dev and < 2.2.0`
 %% `~> 2.0` | `>= 2.0.0 and < 3.0.0`
 %% `~> 2.1` | `>= 2.1.0 and < 3.0.0`
-find_highest_matching(Dep, Constraint, T) ->
-    case ets:lookup(T, Dep) of
-        [{Dep, [[Vsn]]}] ->
-            case ec_semver:pes(Vsn, Constraint) of
-                true ->
-                    {ok, Vsn};
-                false ->
-                    ?WARN("Only existing version of ~s is ~s which does not match constraint ~~> ~s. "
-                         "Using anyway, but it is not guarenteed to work.", [Dep, Vsn, Constraint]),
-                    {ok, Vsn}
-            end;
-        [{Dep, [[HeadVsn | VsnTail]]}] ->
-            {ok, lists:foldl(fun(Version, Highest) ->
-                                case ec_semver:pes(Version, Constraint) andalso
-                                    ec_semver:gt(Version, Highest) of
-                                    true ->
-                                        Version;
-                                    false ->
-                                        Highest
-                                end
-                        end, HeadVsn, VsnTail)};
+find_highest_matching(Dep, Constraint, Table, State) ->
+    verify_table(State),
+    case ets:lookup_element(Table, Dep, 2) of
+        [[HeadVsn | VsnTail]] ->
+            {ok, handle_vsns(Constraint, HeadVsn, VsnTail)};
+        [[Vsn]] ->
+            handle_single_vsn(Dep, Vsn, Constraint);
+        [Vsn] ->
+            handle_single_vsn(Dep, Vsn, Constraint);
+        [HeadVsn | VsnTail] ->
+            {ok, handle_vsns(Constraint, HeadVsn, VsnTail)};
         [] ->
-            ?WARN("Missing registry entry for package ~s", [Dep]),
+            ?WARN("Missing registry entry for package ~s. Try to fix with `rebar3 update`", [Dep]),
             none
     end.
+
+handle_vsns(Constraint, HeadVsn, VsnTail) ->
+    lists:foldl(fun(Version, Highest) ->
+                        case ec_semver:pes(Version, Constraint) andalso
+                            ec_semver:gt(Version, Highest) of
+                            true ->
+                                Version;
+                            false ->
+                                Highest
+                        end
+                end, HeadVsn, VsnTail).
+
+handle_single_vsn(Dep, Vsn, Constraint) ->
+    case ec_semver:pes(Vsn, Constraint) of
+        true ->
+            {ok, Vsn};
+        false ->
+            ?WARN("Only existing version of ~s is ~s which does not match constraint ~~> ~s. "
+                 "Using anyway, but it is not guarenteed to work.", [Dep, Vsn, Constraint]),
+            {ok, Vsn}
+    end.
+
+format_error({missing_package, Package, Version}) ->
+    io_lib:format("Package not found in registry: ~s-~s. Try to fix with `rebar3 update`", [Package, Version]).
+
+verify_table(State) ->
+    ets:info(?PACKAGE_TABLE, named_table) =:= true orelse ?MODULE:load_and_verify_version(State).

src/rebar_plugins.erl

@@ -24,12 +24,12 @@ project_apps_install(State) ->
                         StateAcc1 = handle_plugins(Profile, Plugins, StateAcc),
 
                         lists:foldl(fun(App, StateAcc2) ->
-                                              AppDir = rebar_app_info:dir(App),
-                                              C = rebar_config:consult(AppDir),
-                                              S = rebar_state:new(rebar_state:new(), C, AppDir),
-                                              Plugins2 = rebar_state:get(S, {plugins, Profile}, []),
-                                              handle_plugins(Profile, Plugins2, StateAcc2)
-                                      end, StateAcc1, ProjectApps)
+                                            AppDir = rebar_app_info:dir(App),
+                                            C = rebar_config:consult(AppDir),
+                                            S = rebar_state:new(rebar_state:new(), C, AppDir),
+                                            Plugins2 = rebar_state:get(S, {plugins, Profile}, []),
+                                            handle_plugins(Profile, Plugins2, StateAcc2)
+                                    end, StateAcc1, ProjectApps)
                 end, State, Profiles).
 
 -spec install(rebar_state:t()) -> rebar_state:t().

src/rebar_prv_install_deps.erl

@@ -36,8 +36,6 @@
 -include_lib("providers/include/providers.hrl").
 
 -export([handle_deps_as_profile/4,
-         parse_deps/5,
-         parse_deps/6,
          profile_dep_dir/2,
          find_cycles/1,
          cull_compile/2]).
@@ -125,17 +123,9 @@ handle_deps_as_profile(Profile, State, Deps, Upgrade) ->
     Locks = [],
     Level = 0,
     DepsDir = profile_dep_dir(State, Profile),
-    {SrcDeps, PkgDeps} = parse_deps(DepsDir, Deps, State, Locks, Level),
-    AllSrcProfileDeps = [{Profile, SrcDeps, Locks, Level}],
-    AllPkgProfileDeps = case PkgDeps of
-                            [] ->
-                                [];
-                            _ ->
-                                [{Profile, Level, PkgDeps}]
-                        end,
-    {AllApps, PkgDeps1, Seen, State1} = handle_profile_level(AllSrcProfileDeps, AllPkgProfileDeps, Locks, sets:new(), Upgrade, State),
-
-    handle_profile_pkg_level(PkgDeps1, AllApps, Seen, Upgrade, [], State1).
+    Deps1 = rebar_app_utils:parse_deps(DepsDir, Deps, State, Locks, Level),
+    ProfileLevelDeps = [{Profile, Deps1, Level}],
+    handle_profile_level(ProfileLevelDeps, [], sets:new(), Upgrade, Locks, State).
 
 %% ===================================================================
 %% Internal functions
@@ -144,71 +134,30 @@ handle_deps_as_profile(Profile, State, Deps, Upgrade) ->
 %% finds all the deps in `{deps, ...}` for each profile provided.
 deps_per_profile(Profiles, Upgrade, State) ->
     Level = 0,
-    {AllProfileDeps, PkgDeps} = lists:foldl(fun(Profile, {SrcAcc, PkgAcc}) ->
-                                                case parse_profile_deps(State, Profile, Level) of
-                                                    {Src, {_, _, []}} ->
-                                                        {[Src | SrcAcc], PkgAcc};
-                                                    {Src, Pkg} ->
-                                                        {[Src | SrcAcc], [Pkg | PkgAcc]}
-                                                end
-                                            end, {[], []}, Profiles),
-    {AllApps, PkgDeps1, Seen, State1} = handle_profile_level(AllProfileDeps, PkgDeps, [], sets:new(), Upgrade, State),
     Locks = rebar_state:get(State, {locks, default}, []),
-    handle_profile_pkg_level(PkgDeps1, AllApps, Seen, Upgrade, Locks, State1).
+    Deps = lists:foldl(fun(Profile, DepAcc) ->
+                               [parsed_profile_deps(State, Profile, Level) | DepAcc]
+                       end, [], Profiles),
+    handle_profile_level(Deps, [], sets:new(), Upgrade, Locks, State).
 
-parse_profile_deps(State, Profile, Level) ->
-    Locks = rebar_state:get(State, {locks, Profile}, []),
-    {SrcDeps, PkgDeps} = rebar_state:get(State, {parsed_deps, Profile}, {[], []}),
-    {{Profile, SrcDeps, Locks, Level}, {Profile, Level, PkgDeps}}.
+parsed_profile_deps(State, Profile, Level) ->
+    ParsedDeps = rebar_state:get(State, {parsed_deps, Profile}, []),
+    {Profile, ParsedDeps, Level}.
 
 %% Level-order traversal of all dependencies, across profiles.
 %% If profiles x,y,z are present, then the traversal will go:
 %% x0, y0, z0, x1, y1, z1, ..., xN, yN, zN.
-handle_profile_level([], PkgDeps, SrcApps, Seen, _Upgrade, State) ->
-    {SrcApps, PkgDeps, Seen, State};
-handle_profile_level([{Profile, SrcDeps, Locks, Level} | Rest], PkgDeps, SrcApps, Seen, Upgrade, State) ->
-    {SrcDeps1, PkgDeps1, SrcApps1, State1, Seen1, Locks1} =
-        update_src_deps(Profile, Level, SrcDeps, [], SrcApps
-                        ,State, Upgrade, Seen, Locks),
-    SrcDeps2 = case SrcDeps1 of
+handle_profile_level([], Apps, _Seen, _Upgrade, _Locks, State) ->
+    {Apps, State};
+handle_profile_level([{Profile, Deps, Level} | Rest], 
Apps, Seen, Upgrade, Locks, State) ->
+    {Deps1, Apps1, State1, Seen1} =
+        update_deps(Profile, Level, Deps, Apps
+                   ,State, Upgrade, Seen, Locks),
+    Deps2 = case Deps1 of
         [] -> Rest;
-        _ -> Rest ++ [{Profile, SrcDeps1, Locks1, Level+1}]
+        _ -> Rest ++ [{Profile, Deps1, Level+1}]
     end,
-    handle_profile_level(SrcDeps2, case PkgDeps1 of
-                                       [] ->
-                                           PkgDeps;
-                                       _ ->
-                                           [{Profile, Level+1, PkgDeps1} | PkgDeps]
-                                   end, SrcApps1, sets:union(Seen, Seen1), Upgrade, State1).
-
-handle_profile_pkg_level([], AllApps, _Seen, _Upgrade, _Locks, State) ->
-    {AllApps, State};
-handle_profile_pkg_level(PkgDeps, AllApps, Seen, Upgrade, Locks, State) ->
-    %% Read in package index and dep graph
-    {Packages, Graph} = rebar_state:packages(State),
-    Registry = rebar_state:registry(State),
-    State1 = rebar_state:packages(rebar_state:registry(State, Registry)
-                                 ,{Packages, Graph}),
-
-    S = case rebar_digraph:cull_deps(Graph, lists:keysort(2, PkgDeps), Seen) of
-            {ok, [], []} ->
-                throw({rebar_digraph, no_solution});
-            {ok, [], Discarded} ->
-                [warn_skip_pkg(Pkg, State) || Upgrade =:= false,
-                                              Pkg <- Discarded,
-                                              not(pkg_locked(Pkg, Locks))],
-                [];
-            {ok, Solution, []} ->
-                Solution;
-            {ok, Solution, Discarded} ->
-                [warn_skip_pkg(Pkg, State) || Upgrade =:= false,
-                                              Pkg <- Discarded,
-                                              not(pkg_locked(Pkg, Locks))],
-                Solution
-        end,
-
-    {PkgApps, State2} = update_pkg_deps(S, Packages, Upgrade, Seen, State1, Locks),
-    {AllApps++PkgApps, State2}.
+    handle_profile_level(Deps2, Apps1, sets:union(Seen, Seen1), Upgrade, Locks, State1).
 
 find_cycles(Apps) ->
     case rebar_digraph:compile_order(Apps) of
@@ -220,38 +169,6 @@ find_cycles(Apps) ->
 cull_compile(TopSortedDeps, ProjectApps) ->
     lists:dropwhile(fun not_needs_compile/1, TopSortedDeps -- ProjectApps).
 
-pkg_locked({_, Name, _, _}, Locks) ->
-    pkg_locked(Name, Locks);
-pkg_locked({Name, _}, Locks) ->
-    pkg_locked(Name, Locks);
-pkg_locked(Name, Locks) ->
-    false =/= lists:keyfind(Name, 1, Locks).
-
-update_pkg_deps(Pkgs, Packages, Upgrade, Seen, State, Locks) ->
-    {Solved, _, State1}
-        = lists:foldl(fun({Profile, Pkg}, {Acc, SeenAcc, StateAcc}) ->
-                        DepsDir = profile_dep_dir(State, Profile),
-                        handle_pkg_dep(Profile, Pkg, Packages, Upgrade, DepsDir, Acc, SeenAcc, Locks, StateAcc)
-                      end, {[], Seen, State}, Pkgs),
-    {Solved, State1}.
-
-handle_pkg_dep(Profile, Pkg, Packages, Upgrade, DepsDir, Fetched, Seen, Locks, State) ->
-    IsLock = pkg_locked(Pkg, Locks),
-    AppInfo = package_to_app(DepsDir, Packages, Pkg, IsLock, State),
-    case sets:is_element(rebar_app_info:name(AppInfo), Seen) of
-        true ->
-            {Fetched, Seen, State};
-        false ->
-            Deps = rebar_app_info:deps(AppInfo),
-            Level = rebar_app_info:dep_level(AppInfo),
-            {NewSeen, NewState} = maybe_lock(Profile, AppInfo, Seen, State, Level),
-            {_, AppInfo1} = maybe_fetch(AppInfo, Profile, Upgrade, Seen, NewState),
-            {AppInfo2, _, _, _, _} =
-                handle_dep(NewState, Profile, DepsDir, AppInfo1, Locks, Level),
-            AppInfo3 = rebar_app_info:deps(AppInfo2, Deps),
-            {[AppInfo3 | Fetched], NewSeen, NewState}
-    end.
-
 maybe_lock(Profile, AppInfo, Seen, State, Level) ->
     Name = rebar_app_info:name(AppInfo),
     case rebar_app_info:is_checkout(AppInfo) of
@@ -278,49 +195,28 @@ maybe_lock(Profile, AppInfo, Seen, State, Level) ->
             {sets:add_element(Name, Seen), State}
     end.
 
-package_to_app(DepsDir, Packages, {Parent, Name, Vsn, Level}, IsLock, State) ->
-    case dict:find({Name, Vsn}, Packages) of
-        error ->
-            case rebar_packages:check_registry(Name, Vsn, State) of
-                true ->
-                    throw(?PRV_ERROR({not_rebar_package, Name, Vsn}));
-                false ->
-                    throw(?PRV_ERROR({missing_package, Name, Vsn}))
-            end;
-        {ok, PkgDeps} ->
-            Source = {pkg, Name, Vsn},
-            AppInfo = new_dep(root, DepsDir, Name, Vsn, Source, IsLock, State),
-            AppInfo1 = rebar_app_info:dep_level(rebar_app_info:deps(AppInfo, PkgDeps), Level),
-            BaseDir = rebar_state:get(State, base_dir, []),
-            AppState1 = rebar_state:set(rebar_app_info:state(AppInfo1), base_dir, BaseDir),
-            rebar_app_info:parent(rebar_app_info:state(AppInfo1, AppState1), Parent)
-    end.
-
--spec update_src_deps(atom(), non_neg_integer(), list(), list(), list(), rebar_state:t(), boolean(), sets:set(binary()), list()) -> {list(), list(), list(), rebar_state:t(), sets:set(binary()), list()}.
-update_src_deps(Profile, Level, SrcDeps, PkgDeps, SrcApps, State, Upgrade, Seen, Locks) ->
+update_deps(Profile, Level, Deps, Apps, State, Upgrade, Seen, Locks) ->
     lists:foldl(
-      fun(AppInfo, {SrcDepsAcc, PkgDepsAcc, SrcAppsAcc, StateAcc, SeenAcc, LocksAcc}) ->
-              update_src_dep(AppInfo, Profile, Level,
-                             SrcDepsAcc, PkgDepsAcc, SrcAppsAcc, StateAcc,
-                             Upgrade, SeenAcc, Locks, LocksAcc)
+      fun(AppInfo, {DepsAcc, AppsAcc, StateAcc, SeenAcc}) ->
+              update_dep(AppInfo, Profile, Level,
+                         DepsAcc, AppsAcc, StateAcc,
+                         Upgrade, SeenAcc, Locks)
       end,
-      {[], PkgDeps, SrcApps, State, Seen, Locks},
-      rebar_utils:sort_deps(SrcDeps)).
+      {[], Apps, State, Seen},
+      rebar_utils:sort_deps(Deps)).
 
-
-update_src_dep(AppInfo, Profile, Level, SrcDeps, PkgDeps, SrcApps, State, Upgrade, Seen, BaseLocks, Locks) ->
+update_dep(AppInfo, Profile, Level, Deps, Apps, State, Upgrade, Seen, Locks) ->
     %% If not seen, add to list of locks to write out
     Name = rebar_app_info:name(AppInfo),
     case sets:is_element(Name, Seen) of
         true ->
-            update_seen_src_dep(AppInfo, Profile, Level,
-                                SrcDeps, PkgDeps, SrcApps,
-                                State, Upgrade, Seen, BaseLocks, Locks);
+            update_seen_dep(AppInfo, Profile, Level,
+                            Deps, Apps,
+                            State, Upgrade, Seen, Locks);
         false ->
-            update_unseen_src_dep(AppInfo, Profile, Level,
-                                  SrcDeps, PkgDeps, SrcApps,
-                                  State, Upgrade, Seen, Locks)
-
+            update_unseen_dep(AppInfo, Profile, Level,
+                              Deps, Apps,
+                              State, Upgrade, Seen, Locks)
     end.
 
 profile_dep_dir(State, Profile) ->
@@ -329,45 +225,34 @@ profile_dep_dir(State, Profile) ->
         _ -> rebar_dir:deps_dir(State)
     end.
 
-update_seen_src_dep(AppInfo, _Profile, _Level, SrcDeps, PkgDeps, SrcApps, State, Upgrade, Seen, BaseLocks, Locks) ->
+update_seen_dep(AppInfo, _Profile, _Level, Deps, Apps, State, Upgrade, Seen, Locks) ->
     Name = rebar_app_info:name(AppInfo),
     %% If seen from lock file or user requested an upgrade
     %% don't print warning about skipping
-    case lists:keymember(Name, 1, BaseLocks) of
+    case lists:keymember(Name, 1, Locks) of
         false when Upgrade -> ok;
         false when not Upgrade -> warn_skip_deps(AppInfo, State);
         true -> ok
     end,
-    {SrcDeps, PkgDeps, SrcApps, State, Seen, Locks}.
+    {Deps, Apps, State, Seen}.
 
-update_unseen_src_dep(AppInfo, Profile, Level, SrcDeps, PkgDeps, SrcApps, State, Upgrade, Seen, Locks) ->
+update_unseen_dep(AppInfo, Profile, Level, Deps, Apps, State, Upgrade, Seen, Locks) ->
     {NewSeen, State1} = maybe_lock(Profile, AppInfo, Seen, State, Level),
     {_, AppInfo1} = maybe_fetch(AppInfo, Profile, Upgrade, Seen, State1),
-    {NewSrcDeps, NewPkgDeps, NewSrcApps, State2, NewLocks} =
-        handle_dep(AppInfo1, Profile, SrcDeps, PkgDeps, SrcApps,
-                   Level, State1, Locks),
-    {NewSrcDeps, NewPkgDeps, NewSrcApps, State2, NewSeen, NewLocks}.
-
-handle_dep(AppInfo, Profile, SrcDeps, PkgDeps, SrcApps, Level, State, Locks) ->
     DepsDir = profile_dep_dir(State, Profile),
-    {AppInfo1, NewSrcDeps, NewPkgDeps, NewLocks, State1} =
-        handle_dep(State, Profile, DepsDir, AppInfo, Locks, Level),
-    AppInfo2 = rebar_app_info:dep_level(AppInfo1, Level),
-    {NewSrcDeps ++ SrcDeps
-    ,NewPkgDeps++PkgDeps
-    ,[AppInfo2 | SrcApps]
-    ,State1
-    ,NewLocks}.
-
--spec handle_dep(rebar_state:t(), atom(), file:filename_all(), rebar_app_info:t(), list(), integer()) ->
-                        {rebar_app_info:t(), [rebar_app_info:t()], [pkg_dep()], [integer()], rebar_state:t()}.
+    {AppInfo2, NewDeps, State2} =
+        handle_dep(State1, Profile, DepsDir, AppInfo1, Locks, Level),
+    AppInfo3 = rebar_app_info:dep_level(AppInfo2, Level),
+    {NewDeps ++ Deps, [AppInfo3 | Apps], State2, NewSeen}.
+
+-spec handle_dep(rebar_state:t(), atom(), file:filename_all(), rebar_app_info:t(), list(), integer()) -> {rebar_app_info:t(), [rebar_app_info:t()], [pkg_dep()], [integer()], rebar_state:t()}.
 handle_dep(State, Profile, DepsDir, AppInfo, Locks, Level) ->
     Profiles = rebar_state:current_profiles(State),
     Name = rebar_app_info:name(AppInfo),
+    Vsn = rebar_app_info:original_vsn(AppInfo),
 
     %% Deps may be under a sub project app, find it and use its state if so
-    S0 = rebar_app_info:state(AppInfo),
-    S = rebar_state:registry(S0, rebar_state:registry(State)),
+    S = rebar_app_info:state(AppInfo),
     C = rebar_config:consult(rebar_app_info:dir(AppInfo)),
     S1 = rebar_state:new(S, C, rebar_app_info:dir(AppInfo)),
     S2 = rebar_state:apply_overrides(S1, Name),
@@ -375,23 +260,26 @@ handle_dep(State, Profile, DepsDir, AppInfo, Locks, Level) ->
     S3 = rebar_state:apply_profiles(S2, Profiles),
     Plugins = rebar_state:get(S3, plugins, []),
     S4 = rebar_state:set(S3, {plugins, Profile}, Plugins),
-    AppInfo1 = rebar_app_info:state(AppInfo, S4),
 
     rebar_utils:check_min_otp_version(rebar_state:get(S4, minimum_otp_vsn, undefined)),
     rebar_utils:check_blacklisted_otp_versions(rebar_state:get(S4, blacklisted_otp_vsns, [])),
 
     %% Dep may have plugins to install. Find and install here.
     S5 = rebar_plugins:install(S4),
-    AppInfo2 = rebar_app_info:state(AppInfo1, S5),
+    AppInfo1 = rebar_app_info:state(AppInfo, S5),
 
     %% Upgrade lock level to be the level the dep will have in this dep tree
-    Deps = rebar_state:get(S5, {deps, default}, []),
-    NewLocks = Locks++[{DepName, Source, LockLevel+Level} ||
-                          {DepName, Source, LockLevel} <- rebar_state:get(S5, {locks, default}, [])],
-    AppInfo3 = rebar_app_info:deps(AppInfo2, rebar_state:deps_names(Deps)),
-    {SrcDeps, PkgDeps} = parse_deps(rebar_app_info:name(AppInfo3), DepsDir, Deps
-                                   ,S5, NewLocks, Level+1),
-    {AppInfo3, SrcDeps, PkgDeps, NewLocks, State}.
+    case rebar_app_info:resource_type(AppInfo1) of
+        pkg ->
+            NewDeps = rebar_packages:deps(Name, Vsn, S5),
+            NewDeps1 = rebar_app_utils:parse_deps(Name, DepsDir, NewDeps, S5, Locks, Level+1),
+            {rebar_app_info:deps(AppInfo1, NewDeps), NewDeps1, State};
+        _ ->
+            Deps = rebar_state:get(S5, {deps, default}, []),
+            AppInfo2 = rebar_app_info:deps(AppInfo1, rebar_state:deps_names(Deps)),
+            Deps1 = rebar_app_utils:parse_deps(Name, DepsDir, Deps, S5, Locks, Level+1),
+            {AppInfo2, Deps1, State}
+    end.
 
 -spec maybe_fetch(rebar_app_info:t(), atom(), boolean(),
                   sets:set(binary()), rebar_state:t()) -> {boolean(), rebar_app_info:t()}.
@@ -406,20 +294,20 @@ maybe_fetch(AppInfo, Profile, Upgrade, Seen, State) ->
                 false ->
                     true = fetch_app(AppInfo, AppDir, State),
                     maybe_symlink_default(State, Profile, AppDir, AppInfo),
-                    {true, update_app_info(AppDir, AppInfo)};
+                    {true, rebar_app_info:valid(update_app_info(AppDir, AppInfo), false)};
                 {true, AppInfo1} ->
                     %% Preserve the state we created with overrides
+                    AppInfo2 = copy_app_info(AppInfo, AppInfo1),
                     AppState = rebar_app_info:state(AppInfo),
-                    Parent = rebar_app_info:parent(AppInfo),
-                    Source = rebar_app_info:source(AppInfo),
-                    AppInfo2 = rebar_app_info:parent(rebar_app_info:state(AppInfo1, AppState), Parent),
-                    AppInfo3 = rebar_app_info:source(AppInfo2, Source),
-                    case sets:is_element(rebar_app_info:name(AppInfo), Seen) of
+                    AppInfo3 = rebar_app_info:state(AppInfo2, AppState),
+                    case sets:is_element(rebar_app_info:name(AppInfo3), Seen) of
                         true ->
                             {false, AppInfo3};
                         false ->
                             maybe_symlink_default(State, Profile, AppDir, AppInfo3),
-                            {maybe_upgrade(AppInfo, AppDir, Upgrade, State), AppInfo3}
+                            MaybeUpgrade = maybe_upgrade(AppInfo, AppDir, Upgrade, State),
+                            AppInfo4 = update_app_info(AppDir, AppInfo3),
+                            {MaybeUpgrade, AppInfo4}
                     end
             end
     end.
@@ -467,106 +355,6 @@ make_relative_to_root(State, Path) when is_list(Path) ->
     Root = rebar_dir:root_dir(State),
     rebar_dir:make_relative_path(Path, Root).
 
--spec parse_deps(binary(), list(), rebar_state:t(), list(), integer()) -> {[rebar_app_info:t()], [tuple()]}.
-parse_deps(DepsDir, Deps, State, Locks, Level) ->
-    parse_deps(root, DepsDir, Deps, State, Locks, Level).
-
-parse_deps(Parent, DepsDir, Deps, State, Locks, Level) ->
-    lists:foldl(fun(Dep, Acc) ->
-                        Name = case Dep of
-                                   Dep when is_tuple(Dep) ->
-                                       element(1, Dep);
-                                   Dep ->
-                                       Dep
-                               end,
-                        case lists:keyfind(ec_cnv:to_binary(Name), 1, Locks) of
-                            false ->
-                                parse_dep(Parent, Dep, Acc, DepsDir, false, State);
-                            LockedDep ->
-                                LockedLevel = element(3, LockedDep),
-                                case LockedLevel > Level of
-                                    true ->
-                                        parse_dep(Parent, Dep, Acc, DepsDir, false, State);
-                                    false ->
-                                        parse_dep(Parent, LockedDep, Acc, DepsDir, true, State)
-                                end
-                        end
-                end, {[], []}, Deps).
-
-parse_dep(Parent, {Name, Vsn}, {SrcDepsAcc, PkgDepsAcc}, DepsDir, IsLock, State) when is_list(Vsn) ->
-    %% Versioned Package dependency
-    CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)),
-    case rebar_app_info:discover(CheckoutsDir) of
-        {ok, _App} ->
-            Dep = new_dep(root, DepsDir, Name, [], [], IsLock, State),
-            {[Dep | SrcDepsAcc], PkgDepsAcc};
-        not_found ->
-            {SrcDepsAcc, [parse_goal(Parent
-                                    ,ec_cnv:to_binary(Name)
-                                    ,ec_cnv:to_binary(Vsn)) | PkgDepsAcc]}
-    end;
-parse_dep(Parent, Name, {SrcDepsAcc, PkgDepsAcc}, DepsDir, IsLock, State) when is_atom(Name); is_binary(Name) ->
-    %% Unversioned package dependency
-    {PkgName, PkgVsn} = get_package(ec_cnv:to_binary(Name), State),
-    CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)),
-    case rebar_app_info:discover(CheckoutsDir) of
-        {ok, _App} ->
-            Dep = new_dep(root, DepsDir, Name, [], [], IsLock, State),
-            {[Dep | SrcDepsAcc], PkgDepsAcc};
-        not_found ->
-            {SrcDepsAcc, [{Parent, PkgName, PkgVsn} | PkgDepsAcc]}
-    end;
-parse_dep(Parent, {Name, Source}, {SrcDepsAcc, PkgDepsAcc}, DepsDir, IsLock, State) when is_tuple(Source) ->
-    Dep = new_dep(Parent, DepsDir, Name, [], Source, IsLock, State),
-    {[Dep | SrcDepsAcc], PkgDepsAcc};
-parse_dep(Parent, {Name, _Vsn, Source}, {SrcDepsAcc, PkgDepsAcc}, DepsDir, IsLock, State) when is_tuple(Source) ->
-    Dep = new_dep(Parent, DepsDir, Name, [], Source, IsLock, State),
-    {[Dep | SrcDepsAcc], PkgDepsAcc};
-parse_dep(Parent, {Name, _Vsn, Source, Opts}, {SrcDepsAcc, PkgDepsAcc}, DepsDir, IsLock, State) when is_tuple(Source) ->
-    ?WARN("Dependency option list ~p in ~p is not supported and will be ignored", [Opts, Name]),
-    Dep = new_dep(Parent, DepsDir, Name, [], Source, IsLock, State),
-    {[Dep | SrcDepsAcc], PkgDepsAcc};
-parse_dep(Parent, {_Name, {pkg, Name, Vsn}, Level}, {SrcDepsAcc, PkgDepsAcc}, DepsDir, IsLock, State) when is_integer(Level) ->
-    CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)),
-    case rebar_app_info:discover(CheckoutsDir) of
-        {ok, _App} ->
-            Dep = new_dep(root, DepsDir, Name, [], [], IsLock, State),
-            {[Dep | SrcDepsAcc], PkgDepsAcc};
-        not_found ->
-            {SrcDepsAcc, [{Parent, Name, Vsn} | PkgDepsAcc]}
-    end;
-parse_dep(Parent, {Name, Source, Level}, {SrcDepsAcc, PkgDepsAcc}, DepsDir, IsLock, State) when is_tuple(Source)
-                                                                                              , is_integer(Level) ->
-    Dep = new_dep(Parent, DepsDir, Name, [], Source, IsLock, State),
-    {[Dep | SrcDepsAcc], PkgDepsAcc};
-parse_dep(_, Dep, _, _, _, _) ->
-    throw(?PRV_ERROR({parse_dep, Dep})).
-
-
-new_dep(Parent, DepsDir, Name, Vsn, Source, IsLock, State) ->
-    CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)),
-    {ok, Dep} = case rebar_app_info:discover(CheckoutsDir) of
-                    {ok, App} ->
-                        {ok, rebar_app_info:is_checkout(App, true)};
-                    not_found ->
-                        Dir = ec_cnv:to_list(filename:join(DepsDir, Name)),
-                        case rebar_app_info:discover(Dir) of
-                            {ok, App} ->
-                                {ok, App};
-                            not_found ->
-                                rebar_app_info:new(Name, Vsn,
-                                                   ec_cnv:to_list(filename:join(DepsDir, Name)))
-                        end
-                end,
-    C = rebar_config:consult(rebar_app_info:dir(Dep)),
-    S = rebar_state:new(rebar_state:new(), C, rebar_app_info:dir(Dep)),
-    Overrides = rebar_state:get(State, overrides, []),
-    ParentOverrides = rebar_state:overrides(State),
-    Dep1 = rebar_app_info:state(Dep,
-                               rebar_state:overrides(S, ParentOverrides++Overrides)),
-    AppInfo = rebar_app_info:is_lock(rebar_app_info:source(Dep1, Source), IsLock),
-    rebar_app_info:parent(AppInfo, Parent).
-
 fetch_app(AppInfo, AppDir, State) ->
     ?INFO("Fetching ~s (~p)", [rebar_app_info:name(AppInfo), rebar_app_info:source(AppInfo)]),
     Source = rebar_app_info:source(AppInfo),
@@ -577,15 +365,24 @@ fetch_app(AppInfo, AppDir, State) ->
 %% be read in an parsed.
 update_app_info(AppDir, AppInfo) ->
     {ok, Found} = rebar_app_info:discover(AppDir),
-    Parent = rebar_app_info:parent(AppInfo),
-    Source = rebar_app_info:source(AppInfo),
     AppDetails = rebar_app_info:app_details(Found),
+    Vsn = rebar_app_info:original_vsn(Found),
     Applications = proplists:get_value(applications, AppDetails, []),
     IncludedApplications = proplists:get_value(included_applications, AppDetails, []),
-    AppInfo1 = rebar_app_info:applications(
+    AppInfo1 = rebar_app_info:original_vsn(rebar_app_info:applications(
                  rebar_app_info:app_details(AppInfo, AppDetails),
-                 IncludedApplications++Applications),
-    rebar_app_info:source(rebar_app_info:parent(rebar_app_info:valid(AppInfo1, false), Parent), Source).
+                 IncludedApplications++Applications), Vsn),
+    AppInfo2 = copy_app_info(AppInfo, AppInfo1),
+    rebar_app_info:valid(AppInfo2, undefined).
+
+copy_app_info(OldAppInfo, NewAppInfo) ->
+    ResourceType = rebar_app_info:resource_type(OldAppInfo),
+    Parent = rebar_app_info:parent(OldAppInfo),
+    Source = rebar_app_info:source(OldAppInfo),
+
+    rebar_app_info:resource_type(
+      rebar_app_info:source(
+        rebar_app_info:parent(NewAppInfo, Parent), Source), ResourceType).
 
 maybe_upgrade(AppInfo, AppDir, Upgrade, State) ->
     Source = rebar_app_info:source(AppInfo),
@@ -608,17 +405,6 @@ maybe_upgrade(AppInfo, AppDir, Upgrade, State) ->
             false
     end.
 
--spec parse_goal(binary() | root, binary(), binary()) -> {binary(), binary()} | {binary(), binary(), binary()}.
-parse_goal(Parent, Name, Constraint) ->
-    case re:run(Constraint, "([^\\d]*)(\\d.*)", [{capture, [1,2], binary}]) of
-        {match, [<<>>, Vsn]} ->
-            {Parent, Name, Vsn};
-        {match, [Op, Vsn]} ->
-            {Parent, Name, Vsn, binary_to_atom(Op, utf8)};
-        nomatch ->
-            throw(?PRV_ERROR({bad_constraint, Name, Constraint}))
-    end.
-
 warn_skip_deps(AppInfo, State) ->
     Msg = "Skipping ~s (from ~p) as an app of the same name "
           "has already been fetched",
@@ -629,25 +415,7 @@ warn_skip_deps(AppInfo, State) ->
         true -> ?ERROR(Msg, Args), ?FAIL
     end.
 
-warn_skip_pkg({Name, Source}, State) ->
-    Msg = "Skipping ~s (version ~s from package index) as an app of the same "
-          "name has already been fetched",
-    Args = [Name, Source],
-    case rebar_state:get(State, deps_error_on_conflict, false) of
-        false -> ?WARN(Msg, Args);
-        true -> ?ERROR(Msg, Args), ?FAIL
-    end.
-
 not_needs_compile(App) ->
     not(rebar_app_info:is_checkout(App))
         andalso rebar_app_info:valid(App)
           andalso rebar_state:has_all_artifacts(rebar_app_info:state(App)) =:= true.
-
-get_package(Dep, State) ->
-    case rebar_state:registry(State) of
-        {ok, T} ->
-            {ok, HighestDepVsn} = rebar_packages:find_highest_matching(Dep, "0", T),
-            {Dep, HighestDepVsn};
-        error ->
-            throw(?PRV_ERROR({load_registry_fail, Dep}))
-    end.

src/rebar_prv_packages.erl

@@ -27,19 +27,17 @@ init(State) ->
 
 -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}.
 do(State) ->
-    {Dict, _} = rebar_state:packages(State),
-    print_packages(Dict),
+    print_packages(),
     {ok, State}.
 
 -spec format_error(any()) -> iolist().
 format_error(load_registry_fail) ->
     "Failed to load package regsitry. Try running 'rebar3 update' to fix".
 
-print_packages(Dict) ->
-    Pkgs = lists:keysort(1, dict:fetch_keys(Dict)),
-    SortedPkgs = lists:foldl(fun({Pkg, Vsn}, Acc) ->
-                                       orddict:append_list(Pkg, [Vsn], Acc)
-                               end, orddict:new(), Pkgs),
+print_packages() ->
+    SortedPkgs = ets:foldl(fun({{Pkg, Vsn}, _}, Acc) ->
+                                   orddict:append_list(Pkg, [Vsn], Acc)
+                           end, orddict:new(), ?PACKAGE_TABLE),
 
     orddict:map(fun(Name, Vsns) ->
                         SortedVsns = lists:sort(fun(A, B) ->

src/rebar_prv_update.erl

@@ -9,21 +9,14 @@
          do/1,
          format_error/1]).
 
+-export([hex_to_index/1]).
+
 -include("rebar.hrl").
 -include_lib("providers/include/providers.hrl").
 
 -define(PROVIDER, update).
 -define(DEPS, []).
 
-%% Ignore warning of digraph opaque type when running dialyzer
--dialyzer({no_opaque, do/1}).
--dialyzer({no_opaque, write_registry/3}).
-
-%% Ignoring the opaque type warning won't stop dialyzer from warning of
-%% no return for functions that had the opaque type warnings
--dialyzer({no_return, do/1}).
--dialyzer({no_return, write_registry/3}).
-
 %% ===================================================================
 %% Public API
 %% ===================================================================
@@ -50,16 +43,15 @@ do(State) ->
         TmpDir = ec_file:insecure_mkdtemp(),
         TmpFile = filename:join(TmpDir, "packages.gz"),
 
-        Url = rebar_state:get(State, rebar_packages_cdn, "https://s3.amazonaws.com/s3.hex.pm/registry.ets.gz"),
+        Url = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_HEX_REGISTRY),
         {ok, _RequestId} = httpc:request(get, {Url, []},
                                          [], [{stream, TmpFile}, {sync, true}],
                                          rebar),
         {ok, Data} = file:read_file(TmpFile),
         Unzipped = zlib:gunzip(Data),
         ok = file:write_file(HexFile, Unzipped),
-        {Dict, Graph} = hex_to_graph(HexFile),
-        write_registry(Dict, Graph, State),
-        true = digraph:delete(Graph),
+
+        hex_to_index(State),
         ok
     catch
         _E:C ->
@@ -73,56 +65,58 @@ do(State) ->
 format_error(package_index_write) ->
     "Failed to write package index.".
 
--spec write_registry(rebar_dict(), {digraph, ets:tid(), ets:tid(), ets:tid(), any()}, rebar_state:t()) -> ok | {error, atom()}.
-write_registry(Dict, {digraph, Edges, Vertices, Neighbors, _}, State) ->
-    RegistryDir = rebar_packages:package_dir(State),
-    filelib:ensure_dir(filename:join(RegistryDir, "dummy")),
-    ets:tab2file(Edges, filename:join(RegistryDir, "edges")),
-    ets:tab2file(Vertices, filename:join(RegistryDir, "vertices")),
-    ets:tab2file(Neighbors, filename:join(RegistryDir, "neighbors")),
-    file:write_file(filename:join(RegistryDir, "dict"), term_to_binary(Dict)).
-
 is_supported(<<"make">>) -> true;
 is_supported(<<"rebar">>) -> true;
+is_supported(<<"rebar3">>) -> true;
 is_supported(_) -> false.
 
-hex_to_graph(Filename) ->
-    {ok, T} = ets:file2tab(Filename),
-    Graph = digraph:new(),
-    ets:foldl(fun({Pkg, [Versions]}, ok) when is_binary(Pkg), is_list(Versions) ->
-                      lists:foreach(fun(Version) ->
-                                            digraph:add_vertex(Graph, {Pkg, Version}, 1)
-                                    end, Versions);
-                 (_, ok) ->
-                      ok
-              end, ok, T),
-
-    Dict1 = ets:foldl(fun({{Pkg, PkgVsn}, [Deps, _, BuildTools | _]}, Dict) when is_list(BuildTools) ->
-                              case lists:any(fun is_supported/1, BuildTools) of
-                                  true ->
-                                      DepsList = update_graph(Pkg, PkgVsn, Deps, T, Graph),
-                                      dict:store({Pkg, PkgVsn}, DepsList, Dict);
-                                  false ->
-                                      Dict
-                              end;
-                         (_, Dict) ->
-                              Dict
-                      end, dict:new(), T),
-    {Dict1, Graph}.
-
-update_graph(Pkg, PkgVsn, Deps, HexRegistry, Graph) ->
+hex_to_index(State) ->
+    RegistryDir = rebar_packages:package_dir(State),
+    HexFile = filename:join(RegistryDir, "registry"),
+    try ets:file2tab(HexFile) of
+        {ok, Registry} ->
+            try
+                (catch ets:delete(?PACKAGE_TABLE)),
+                ets:new(?PACKAGE_TABLE, [named_table, public]),
+                ets:foldl(fun({{Pkg, PkgVsn}, [Deps, Checksum, BuildTools | _]}, _) when is_list(BuildTools) ->
+                                  case lists:any(fun is_supported/1, BuildTools) of
+                                      true ->
+                                          DepsList = update_deps_list(Deps, Registry, State),
+                                          ets:insert(?PACKAGE_TABLE, {{Pkg, PkgVsn}, DepsList, Checksum});
+                                      false ->
+                                          true
+                                  end;
+                             ({Pkg, [Vsns]}, _) when is_binary(Pkg) ->
+                                  ets:insert(?PACKAGE_TABLE, {Pkg, Vsns});
+                             (_, _) ->
+                                  true
+                          end, true, Registry),
+
+                ets:insert(?PACKAGE_TABLE, {package_index_version, ?PACKAGE_INDEX_VERSION}),
+                ets:tab2file(?PACKAGE_TABLE, filename:join(RegistryDir, "packages.idx")),
+                true
+            after
+                catch ets:delete(Registry)
+            end;
+        {error, Reason} ->
+            ?DEBUG("Error loading package registry: ~p", [Reason]),
+            false
+    catch
+        _:_ ->
+            fail
+    end.
+
+update_deps_list(Deps, HexRegistry, State) ->
     lists:foldl(fun([Dep, DepVsn, false, _AppName | _], DepsListAcc) ->
                         case DepVsn of
                             <<"~> ", Vsn/binary>> ->
-                                case rebar_packages:find_highest_matching(Dep, Vsn, HexRegistry) of
+                                case rebar_packages:find_highest_matching(Dep, Vsn, HexRegistry, State) of
                                     {ok, HighestDepVsn} ->
-                                        digraph:add_edge(Graph, {Pkg, PkgVsn}, {Dep, HighestDepVsn}),
-                                        [{Dep, DepVsn} | DepsListAcc];
+                                        [{Dep, HighestDepVsn} | DepsListAcc];
                                     none ->
                                         DepsListAcc
                                 end;
                             Vsn ->
-                                digraph:add_edge(Graph, {Pkg, PkgVsn}, {Dep, Vsn}),
                                 [{Dep, Vsn} | DepsListAcc]
                         end;
                    ([_Dep, _DepVsn, true, _AppName | _], DepsListAcc) ->

src/rebar_prv_upgrade.erl

@@ -54,7 +54,7 @@ do(State) ->
             Deps0 = top_level_deps(Deps, Locks),
             State1 = rebar_state:set(State, {deps, default}, Deps0),
             DepsDir = rebar_prv_install_deps:profile_dep_dir(State, default),
-            D = rebar_prv_install_deps:parse_deps(root, DepsDir, Deps0, State1, Locks0, 0),
+            D = rebar_app_utils:parse_deps(root, DepsDir, Deps0, State1, Locks0, 0),
             State2 = rebar_state:set(State1, {parsed_deps, default}, D),
             State3 = rebar_state:set(State2, {locks, default}, Locks0),
             State4 = rebar_state:set(State3, upgrade, true),
@@ -83,7 +83,6 @@ format_error({transitive_dependency, Name}) ->
 format_error(Reason) ->
     io_lib:format("~p", [Reason]).
 
-
 parse_names(Bin, Locks) ->
     case lists:usort(re:split(Bin, <<" *, *">>, [trim])) of
         %% Nothing submitted, use *all* apps

src/rebar_state.erl

@@ -38,9 +38,6 @@
          overrides/1, overrides/2,
          apply_overrides/2,
 
-         packages/1, packages/2,
-         registry/1, registry/2,
-
          resources/1, resources/2, add_resource/2,
          providers/1, providers/2, add_provider/2]).
 
@@ -65,8 +62,6 @@
                   all_plugin_deps     = []          :: [rebar_app_info:t()],
                   all_deps            = []          :: [rebar_app_info:t()],
 
-                  packages            = undefined   :: {rebar_dict(), rebar_digraph()} | undefined,
-                  registry            = undefined   :: {ok, ets:tid()} | error | undefined,
                   overrides           = [],
                   resources           = [],
                   providers           = []}).
@@ -88,16 +83,9 @@ new(Config) when is_list(Config) ->
     Terms = [{{deps, default}, Deps}, {{plugins, default}, Plugins} | Config],
     true = rebar_config:verify_config_format(Terms),
     Opts = dict:from_list(Terms),
-    load_package_registry(
-      BaseState#state_t { dir = rebar_dir:get_cwd(),
-                          default = Opts,
-                          opts = Opts }).
-
-load_package_registry(Config0) ->
-    Registry = rebar_packages:registry(Config0),
-    Packages = rebar_packages:packages(Config0),
-    Config0#state_t{registry = Registry,
-                    packages = Packages}.
+    BaseState#state_t { dir = rebar_dir:get_cwd(),
+                        default = Opts,
+                        opts = Opts }.
 
 -spec new(t() | atom(), list()) -> t().
 new(Profile, Config) when is_atom(Profile)
@@ -109,11 +97,10 @@ new(Profile, Config) when is_atom(Profile)
     Terms = [{{deps, default}, Deps}, {{plugins, default}, Plugins} | Config],
     true = rebar_config:verify_config_format(Terms),
     Opts = dict:from_list(Terms),
-    load_package_registry(
-      BaseState#state_t { dir = rebar_dir:get_cwd(),
-                          current_profiles = [Profile],
-                          default = Opts,
-                          opts = Opts });
+    BaseState#state_t { dir = rebar_dir:get_cwd(),
+                        current_profiles = [Profile],
+                        default = Opts,
+                        opts = Opts };
 new(ParentState=#state_t{}, Config) ->
     %% Load terms from rebar.config, if it exists
     Dir = rebar_dir:get_cwd(),
@@ -446,22 +433,6 @@ namespace(#state_t{namespace=Namespace}) ->
 namespace(State=#state_t{}, Namespace) ->
     State#state_t{namespace=Namespace}.
 
-packages(#state_t{packages=undefined}) ->
-    throw(packages_usage_error);
-packages(#state_t{packages=Packages}) ->
-    Packages.
-
-packages(State, Packages) ->
-    State#state_t{packages=Packages}.
-
-registry(#state_t{registry=undefined}) ->
-    throw(registry_usage_error);
-registry(#state_t{registry=Registry}) ->
-    Registry.
-
-registry(State, Registry) ->
-    State#state_t{registry=Registry}.
-
 -spec resources(t()) -> [{rebar_resource:type(), module()}].
 resources(#state_t{resources=Resources}) ->
     Resources.

test/mock_pkg_resource.erl

@@ -35,7 +35,7 @@ mock(Opts) ->
 
 unmock() ->
     meck:unload(?MOD),
-    meck:unload(rebar_state).
+    meck:unload(rebar_packages).
 
 %%%%%%%%%%%%%%%
 %%% Private %%%
@@ -111,32 +111,19 @@ mock_pkg_index(Opts) ->
     Deps = proplists:get_value(pkgdeps, Opts, []),
     Skip = proplists:get_value(not_in_index, Opts, []),
     %% Dict: {App, Vsn}: [{<<"link">>, <<>>}, {<<"deps">>, []}]
-    %% Digraph: all apps and deps in the index
+    %% Index: all apps and deps in the index
+
     Dict = find_parts(Deps, Skip),
-    GraphParts = to_graph_parts(Dict),
-    Digraph = rebar_digraph:restore_graph(GraphParts),
-    meck:new(rebar_state, [passthrough, no_link]),
-    meck:expect(rebar_state, registry,
-                fun(_State) -> {ok, to_registry(Deps)} end),
-    meck:expect(rebar_state, packages,
-                fun(_State) -> {Dict, Digraph} end).
+    meck:new(rebar_packages, [passthrough, no_link]),
+    meck:expect(rebar_packages, packages,
+                fun(_State) -> to_index(Deps, Dict) end),
+    meck:expect(rebar_packages, load_and_verify_version,
+                fun(_State) -> to_index(Deps, Dict), true end).
 
 %%%%%%%%%%%%%%%
 %%% Helpers %%%
 %%%%%%%%%%%%%%%
 
-to_registry(Deps) ->
-    Tid = ets:new(registry, []),
-    lists:foreach(fun({{Name, Vsn}, _}) ->
-                          case ets:lookup(Tid, Name) of
-                              [{_, [Vsns]}] ->
-                                  ets:insert(Tid, {Name, [[Vsn | Vsns]]});
-                              _ ->
-                                  ets:insert(Tid, {Name, [[Vsn]]})
-                          end
-                  end, Deps),
-    Tid.
-
 all_files(Dir) ->
     filelib:wildcard(filename:join([Dir, "**"])).
 
@@ -156,10 +143,20 @@ find_parts([{AppName, Deps}|Rest], Skip, Acc) ->
             find_parts(Rest, Skip, AccNew)
     end.
 
-to_graph_parts(Dict) ->
-    LastUpdated = os:timestamp(),
-    dict:fold(fun(K,Deps,{Ks,Vs}) ->
-            {[{K,LastUpdated}|Ks],
-             [{K,{list_to_binary(atom_to_list(DK)), list_to_binary(DV)}}
-             || {DK,DV} <- Deps]  ++ Vs}
-        end, {[],[]}, Dict).
+to_index(AllDeps, Dict) ->
+    catch ets:delete(package_index),
+    ets:new(package_index, [named_table, public]),
+    dict:fold(
+      fun(K, Deps, _) ->
+              DepsList = [{ec_cnv:to_binary(DK), ec_cnv:to_binary(DV)} || {DK, DV} <- Deps],
+              ets:insert(package_index, {K, DepsList, <<"checksum">>})
+      end, ok, Dict),
+    ets:insert(package_index, {package_index_version, 3}),
+    lists:foreach(fun({{Name, Vsn}, _}) ->
+                          case ets:lookup(package_index,  ec_cnv:to_binary(Name)) of
+                              [{_, Vsns}] ->
+                                  ets:insert(package_index, {ec_cnv:to_binary(Name), [ec_cnv:to_binary(Vsn) | Vsns]});
+                              _ ->
+                                  ets:insert(package_index, {ec_cnv:to_binary(Name), [ec_cnv:to_binary(Vsn)]})
+                          end
+                  end, AllDeps).

test/rebar_deps_SUITE.erl

@@ -297,5 +297,5 @@ in_warnings(git, Warns, NameRaw, VsnRaw) ->
 in_warnings(pkg, Warns, NameRaw, VsnRaw) ->
     Name = iolist_to_binary(NameRaw),
     Vsn = iolist_to_binary(VsnRaw),
-    1 =< length([1 || {_, [AppName, AppVsn]} <- Warns,
+    1 =< length([1 || {_, [AppName, {pkg, _, AppVsn}]} <- Warns,
                       AppName =:= Name, AppVsn =:= Vsn]).

test/rebar_install_deps_SUITE.erl

@@ -15,7 +15,7 @@ groups() ->
      {git, [], [{group, unique}]},
      {pkg, [], [{group, unique}]},
      {mixed, [], [
-        m_flat1, m_flat2, m_circular1, m_circular2, m_circular3,
+        m_flat1, m_flat2, m_circular1, m_circular2,
         m_pick_source1, m_pick_source2, m_pick_source3,
         m_pick_source4, m_pick_source5, m_source_to_pkg,
         m_pkg_level1, m_pkg_level2, m_pkg_level3, m_pkg_level3_alpha_order
@@ -96,9 +96,9 @@ format_expected_mdeps(Deps) ->
 format_expected_mixed_warnings(Warnings) ->
     [case W of
         {N, Vsn} when hd(N) >= $a, hd(N) =< $z -> {pkg, string:to_upper(N), Vsn};
-        {N, Vsn} when hd(N) >= $A, hd(N) =< $Z -> {src, N, Vsn};
+        {N, Vsn} when hd(N) >= $A, hd(N) =< $Z -> {git, N, Vsn};
         N when hd(N) >= $a, hd(N) =< $z -> {pkg, string:to_upper(N), "0.0.0"};
-        N when hd(N) >= $A, hd(N) =< $Z -> {src, N, "0.0.0"}
+        N when hd(N) >= $A, hd(N) =< $Z -> {git, N, "0.0.0"}
      end || W <- Warnings].
 
 %% format:
@@ -199,39 +199,32 @@ mdeps(m_circular2) ->
     {[{"B", [{"c", [{"b", []}]}]}],
      [],
      {error, {rebar_prv_install_deps, {cycles, [[<<"B">>,<<"C">>]]}}}};
-mdeps(m_circular3) ->
-    %% Spot the circular dep due to being to low in the deps tree
-    %% but as a source dep, taking precedence over packages
-    {[{"B", [{"C", "2", [{"B", []}]}]},
-      {"c", "1", [{"d",[]}]}],
-     [],
-     {error, {rebar_prv_install_deps, {cycles, [[<<"B">>,<<"C">>]]}}}};
 mdeps(m_pick_source1) ->
     {[{"B", [{"D", []}]},
       {"c", [{"d", []}]}],
      ["d"],
      {ok, ["B", "c", "D"]}};
 mdeps(m_pick_source2) ->
-    {[{"b", [{"d", []}]},
-      {"C", [{"D", []}]}],
-     ["d"],
-     {ok, ["b", "C", "D"]}};
-mdeps(m_pick_source3) ->
     %% The order of declaration is important.
     {[{"b", []},
       {"B", []}],
      [],
      {ok, ["b"]}};
-mdeps(m_pick_source4) ->
+mdeps(m_pick_source3) ->
     {[{"B", []},
       {"b", []}],
      [],
      {ok, ["B"]}};
+mdeps(m_pick_source4) ->
+    {[{"b", [{"d", "1", []}]},
+      {"C", [{"D", "1", []}]}],
+     [{"D", "1"}],
+     {ok, ["b", "C", {"d", "1"}]}};
 mdeps(m_pick_source5) ->
-    {[{"B", [{"d", []}]},
-      {"C", [{"D", []}]}],
-     ["d"],
-     {ok, ["B", "C", "D"]}};
+    {[{"B", [{"d", "1", []}]},
+      {"C", [{"D", "1", []}]}],
+     [{"D", "1"}],
+     {ok, ["B", "C", {"d", "1"}]}};
 mdeps(m_source_to_pkg) ->
     {[{"B", [{"c",[{"d", []}]}]}],
      [],
@@ -438,7 +431,6 @@ m_flat1(Config) -> run(Config).
 m_flat2(Config) -> run(Config).
 m_circular1(Config) -> run(Config).
 m_circular2(Config) -> run(Config).
-m_circular3(Config) -> run(Config).
 m_pick_source1(Config) -> run(Config).
 m_pick_source2(Config) -> run(Config).
 m_pick_source3(Config) -> run(Config).
@@ -483,5 +475,5 @@ in_warnings(git, Warns, NameRaw, VsnRaw) ->
 in_warnings(pkg, Warns, NameRaw, VsnRaw) ->
     Name = iolist_to_binary(NameRaw),
     Vsn = iolist_to_binary(VsnRaw),
-    1 =< length([1 || {_, [AppName, AppVsn]} <- Warns,
+    1 =< length([1 || {_, [AppName, {pkg, _, AppVsn}]} <- Warns,
                       AppName =:= Name, AppVsn =:= Vsn]).

test/rebar_pkg_SUITE.erl

@@ -159,25 +159,33 @@ mock_config(Name, Config) ->
     Priv = ?config(priv_dir, Config),
     CacheRoot = filename:join([Priv, "cache", atom_to_list(Name)]),
     TmpDir = filename:join([Priv, "tmp", atom_to_list(Name)]),
-    T = ets:new(fake_registry, [public]),
-    ets:insert_new(T, [
-        {{<<"badindexchk">>,<<"1.0.0">>}, [[], ?bad_checksum]},
-        {{<<"goodpkg">>,<<"1.0.0">>}, [[], ?good_checksum]},
-        {{<<"badpkg">>,<<"1.0.0">>}, [[], ?good_checksum]}
+    Tid = ets:new(registry_table, [public]),
+    ets:insert_new(Tid, [
+        {<<"badindexchk">>,[[<<"1.0.0">>]]},
+        {<<"goodpkg">>,[[<<"1.0.0">>]]},
+        {<<"badpkg">>,[[<<"1.0.0">>]]},
+        {{<<"badindexchk">>,<<"1.0.0">>}, [[], ?bad_checksum, [<<"rebar3">>]]},
+        {{<<"goodpkg">>,<<"1.0.0">>}, [[], ?good_checksum, [<<"rebar3">>]]},
+        {{<<"badpkg">>,<<"1.0.0">>}, [[], ?good_checksum, [<<"rebar3">>]]}
     ]),
     CacheDir = filename:join([CacheRoot, "hex", "com", "test", "packages"]),
     filelib:ensure_dir(filename:join([CacheDir, "registry"])),
-    ok = ets:tab2file(T, filename:join([CacheDir, "registry"])),
+    ok = ets:tab2file(Tid, filename:join([CacheDir, "registry"])),
+
     %% The state returns us a fake registry
     meck:new(rebar_state, [passthrough]),
-    meck:expect(rebar_state, registry,
-                fun(_State) -> {ok, T} end),
     meck:expect(rebar_state, get,
                 fun(_State, rebar_packages_cdn, _Default) ->
                     "http://test.com/"
                 end),
+
     meck:new(rebar_dir, [passthrough]),
     meck:expect(rebar_dir, global_cache_dir, fun(_) -> CacheRoot end),
+
+    meck:new(rebar_packages, [passthrough]),
+    meck:expect(rebar_packages, package_dir, fun(_) -> CacheDir end),
+    rebar_prv_update:hex_to_index(rebar_state:new()),
+
     %% Cache fetches are mocked -- we assume the server and clients are
     %% correctly used.
     GoodCache = ?config(good_cache, Config),
@@ -194,7 +202,7 @@ mock_config(Name, Config) ->
     [{cache_root, CacheRoot},
      {cache_dir, CacheDir},
      {tmp_dir, TmpDir},
-     {mock_table, T} | Config].
+     {mock_table, Tid} | Config].
 
 unmock_config(Config) ->
     meck:unload(),