replace use of dict of packages and registry with single ets table

src/rebar.hrl

@@ -23,8 +23,14 @@
 -define(DEFAULT_RELEASE_DIR, "rel").
 -define(DEFAULT_CONFIG_FILE, "rebar.config").
 -define(DEFAULT_CDN, "https://s3.amazonaws.com/s3.hex.pm/tarballs").
+-define(DEFAULT_HEX_REGISTRY, "https://s3.amazonaws.com/s3.hex.pm/registry.ets.gz").
 -define(LOCK_FILE, "rebar.lock").
 
+-define(PACKAGE_INDEX_VERSION, 3).
+-define(PACKAGE_TABLE, package_index).
+-define(INDEX_FILE, "packages.idx").
+-define(REGISTRY_FILE, "registry").
+
 -ifdef(namespaced_types).
 -type rebar_dict() :: dict:dict().
 -else.

src/rebar3.erl

@@ -105,6 +105,7 @@ run_aux(State, RawArgs) ->
     {ok, Providers} = application:get_env(rebar, providers),
     %% Providers can modify profiles stored in opts, so set default after initializing providers
     State4 = rebar_state:create_logic_providers(Providers, State3),
+    rebar_packages:packages(State4),
     State5 = rebar_plugins:project_apps_install(State4),
     State6 = rebar_state:default(State5, rebar_state:opts(State5)),
 
@@ -280,9 +281,15 @@ state_from_global_config(Config, GlobalConfigFile) ->
     %% We don't want to worry about global plugin install state effecting later
     %% usage. So we throw away the global profile state used for plugin install.
     GlobalConfigThrowAway = rebar_state:current_profiles(GlobalConfig, [global]),
-    GlobalState = rebar_plugins:handle_plugins(global,
-                                               rebar_state:get(GlobalConfigThrowAway, plugins, []),
-                                               GlobalConfigThrowAway),
+    GlobalState = case rebar_state:get(GlobalConfigThrowAway, plugins, []) of
+                      [] ->
+                          GlobalConfigThrowAway;
+                      GlobalPluginsToInstall ->
+                          rebar_packages:packages(GlobalConfigThrowAway),
+                          rebar_plugins:handle_plugins(global,
+                                                       GlobalPluginsToInstall,
+                                                       GlobalConfigThrowAway)
+                  end,
     GlobalPlugins = rebar_state:providers(GlobalState),
     GlobalConfig2 = rebar_state:set(GlobalConfig, plugins, []),
     GlobalConfig3 = rebar_state:set(GlobalConfig2, {plugins, global}, rebar_state:get(GlobalConfigThrowAway, plugins, [])),

src/rebar_app_utils.erl

@@ -212,13 +212,8 @@ parse_goal(Name, Constraint) ->
     end.
 
 get_package(Dep, State) ->
-    case rebar_state:registry(State) of
-        {ok, T} ->
-            {ok, HighestDepVsn} = rebar_packages:find_highest_matching(Dep, "0", T),
-            {Dep, HighestDepVsn};
-        error ->
-            throw(?PRV_ERROR({load_registry_fail, Dep}))
-    end.
+    {ok, HighestDepVsn} = rebar_packages:find_highest_matching(Dep, "0", ?PACKAGE_TABLE, State),
+    {Dep, HighestDepVsn}.
 
 -spec has_all_beams(file:filename_all(), [module()]) ->
     true | ?PRV_ERROR({missing_module, module()}).

src/rebar_packages.erl

@@ -1,71 +1,60 @@
 -module(rebar_packages).
 
 -export([packages/1
-        ,packages_graph/1
-        ,registry/1
+        ,close_packages/0
+        ,load_and_verify_version/1
+        ,deps/3
         ,package_dir/1
-        ,check_registry/3
         ,registry_checksum/2
-        ,find_highest_matching/3]).
+        ,find_highest_matching/4
+        ,format_error/1]).
 
 -export_type([package/0]).
 
 -include("rebar.hrl").
+-include_lib("providers/include/providers.hrl").
 
 -type pkg_name() :: binary() | atom().
 -type vsn() :: binary().
 -type package() :: pkg_name() | {pkg_name(), vsn()}.
 
--spec packages(rebar_state:t()) -> rebar_dict().
-%% DON'T USE IT! Use rebar_state:packages(State) instead.
+-spec packages(rebar_state:t()) -> ets:tid().
 packages(State) ->
-    RegistryDir = package_dir(State),
-    DictFile = filename:join(RegistryDir, "dict"),
-
-    try
-        {ok, DictBinary} = file:read_file(DictFile),
-        binary_to_term(DictBinary)
-    catch
-        _:_ ->
+    catch ets:delete(?PACKAGE_TABLE),
+    case load_and_verify_version(State) of
+        true ->
+            ok;
+        false ->
+            ?DEBUG("Error loading package index.", []),
             ?ERROR("Bad packages index, try to fix with `rebar3 update`", []),
-            dict:new()
+            ets:new(?PACKAGE_TABLE, [named_table, public])
     end.
 
--spec packages_graph(rebar_state:t()) -> rebar_digraph().
-packages_graph(State) ->
-    RegistryDir = package_dir(State),
-    Edges = filename:join(RegistryDir, "edges"),
-    Vertices = filename:join(RegistryDir, "vertices"),
-    Neighbors = filename:join(RegistryDir, "neighbors"),
+close_packages() ->
+    catch ets:delete(?PACKAGE_TABLE).
 
-    case lists:all(fun(X) -> filelib:is_file(X) end, [Edges, Vertices, Neighbors]) of
-        true ->
-            try
-                {ok, EdgesTab} = ets:file2tab(Edges),
-                {ok, VerticesTab} = ets:file2tab(Vertices),
-                {ok, NeighborsTab} = ets:file2tab(Neighbors),
-                {digraph, EdgesTab, VerticesTab, NeighborsTab, true}
-            catch
-                _:_ ->
-                    ?ERROR("Bad packages index, try to fix with `rebar3 update`", []),
-                    digraph:new()
+load_and_verify_version(State) ->
+    RegistryDir = package_dir(State),
+    case ets:file2tab(filename:join(RegistryDir, ?INDEX_FILE)) of
+        {ok, _} ->
+            case ets:lookup_element(?PACKAGE_TABLE, package_index_version, 2) of
+                ?PACKAGE_INDEX_VERSION ->
+                    true;
+                _ ->
+                    (catch ets:delete(?PACKAGE_TABLE)),
+                    rebar_prv_update:hex_to_index(State)
             end;
-        false ->
-            ?ERROR("Bad packages index, try to fix with `rebar3 update`", []),
-            digraph:new()
+        _ ->
+            rebar_prv_update:hex_to_index(State)
     end.
 
--spec registry(rebar_state:t()) -> {ok, ets:tid()} | {error, any()}.
-%% DON'T USE IT! Use rebar_state:registry(State) instead.
-registry(State) ->
-    RegistryDir = package_dir(State),
-    HexFile = filename:join(RegistryDir, "registry"),
-    case ets:file2tab(HexFile) of
-        {ok, T} ->
-            {ok, T};
-        {error, Reason} ->
-            ?DEBUG("Error loading registry: ~p", [Reason]),
-            error
+deps(Name, Vsn, State) ->
+    try
+        verify_table(State),
+        ets:lookup_element(?PACKAGE_TABLE, {ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)}, 2)
+    catch
+        _:_ ->
+            throw(?PRV_ERROR({missing_package, ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)}))
     end.
 
 package_dir(State) ->
@@ -78,27 +67,13 @@ package_dir(State) ->
     ok = filelib:ensure_dir(filename:join(PackageDir, "placeholder")),
     PackageDir.
 
-
-check_registry(Pkg, Vsn, State) ->
-    case rebar_state:registry(State) of
-        {ok, T} ->
-            case ets:lookup(T, Pkg) of
-                [{Pkg, [Vsns]}] ->
-                    lists:member(Vsn, Vsns);
-                _ ->
-                    false
-            end;
-        error ->
-            false
-    end.
-
 registry_checksum({pkg, Name, Vsn}, State) ->
-    {ok
, Registry} = rebar_state:registry(State),
-    case ets:lookup(Registry, {Name, Vsn}) of
-        [{{_, _}, [_, Checksum | _]}] ->
-            Checksum;
-        [] ->
-            none
+    try
+        verify_table(State),
+        ets:lookup_element(?PACKAGE_TABLE, {Name, Vsn}, 3)
+    catch
+        _:_ ->
+            throw(?PRV_ERROR({missing_package, ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)}))
     end.
 
 %% Hex supports use of ~> to specify the version required for a dependency.
@@ -116,28 +91,45 @@ registry_checksum({pkg, Name, Vsn}, State) ->
 %% `~> 2.1.3-dev` | `>= 2.1.3-dev and < 2.2.0`
 %% `~> 2.0` | `>= 2.0.0 and < 3.0.0`
 %% `~> 2.1` | `>= 2.1.0 and < 3.0.0`
-find_highest_matching(Dep, Constraint, T) ->
-    case ets:lookup(T, Dep) of
-        [{Dep, [[Vsn]]}] ->
-            case ec_semver:pes(Vsn, Constraint) of
-                true ->
-                    {ok, Vsn};
-                false ->
-                    ?WARN("Only existing version of ~s is ~s which does not match constraint ~~> ~s. "
-                         "Using anyway, but it is not guarenteed to work.", [Dep, Vsn, Constraint]),
-                    {ok, Vsn}
-            end;
-        [{Dep, [[HeadVsn | VsnTail]]}] ->
-            {ok, lists:foldl(fun(Version, Highest) ->
-                                case ec_semver:pes(Version, Constraint) andalso
-                                    ec_semver:gt(Version, Highest) of
-                                    true ->
-                                        Version;
-                                    false ->
-                                        Highest
-                                end
-                        end, HeadVsn, VsnTail)};
+find_highest_matching(Dep, Constraint, Table, State) ->
+    verify_table(State),
+    case ets:lookup_element(Table, Dep, 2) of
+        [[HeadVsn | VsnTail]] ->
+            {ok, handle_vsns(Constraint, HeadVsn, VsnTail)};
+        [[Vsn]] ->
+            handle_single_vsn(Dep, Vsn, Constraint);
+        [Vsn] ->
+            handle_single_vsn(Dep, Vsn, Constraint);
+        [HeadVsn | VsnTail] ->
+            {ok, handle_vsns(Constraint, HeadVsn, VsnTail)};
         [] ->
-            ?WARN("Missing registry entry for package ~s", [Dep]),
+            ?WARN("Missing registry entry for package ~s. Try to fix with `rebar3 update`", [Dep]),
             none
     end.
+
+handle_vsns(Constraint, HeadVsn, VsnTail) ->
+    lists:foldl(fun(Version, Highest) ->
+                        case ec_semver:pes(Version, Constraint) andalso
+                            ec_semver:gt(Version, Highest) of
+                            true ->
+                                Version;
+                            false ->
+                                Highest
+                        end
+                end, HeadVsn, VsnTail).
+
+handle_single_vsn(Dep, Vsn, Constraint) ->
+    case ec_semver:pes(Vsn, Constraint) of
+        true ->
+            {ok, Vsn};
+        false ->
+            ?WARN("Only existing version of ~s is ~s which does not match constraint ~~> ~s. "
+                 "Using anyway, but it is not guarenteed to work.", [Dep, Vsn, Constraint]),
+            {ok, Vsn}
+    end.
+
+format_error({missing_package, Package, Version}) ->
+    io_lib:format("Package not found in registry: ~s-~s. Try to fix with `rebar3 update`", [Package, Version]).
+
+verify_table(State) ->
+    ets:info(?PACKAGE_TABLE, named_table) =:= true orelse ?MODULE:load_and_verify_version(State).

src/rebar_plugins.erl

@@ -16,10 +16,12 @@
 
 -spec project_apps_install(rebar_state:t()) -> rebar_state:t().
 project_apps_install(State) ->
+    rebar_packages:packages(State),
+
     Profiles = rebar_state:current_profiles(State),
     ProjectApps = rebar_state:project_apps(State),
 
-    lists:foldl(fun(Profile, StateAcc) ->
+    State1 = lists:foldl(fun(Profile, StateAcc) ->
                         Plugins = rebar_state:get(State, {plugins, Profile}, []),
                         StateAcc1 = handle_plugins(Profile, Plugins, StateAcc),
 
@@ -30,7 +32,11 @@ project_apps_install(State) ->
                                               Plugins2 = rebar_state:get(S, {plugins, Profile}, []),
                                               handle_plugins(Profile, Plugins2, StateAcc2)
                                       end, StateAcc1, ProjectApps)
-                end, State, Profiles).
+                end, State, Profiles),
+
+    rebar_packages:close_packages(),
+
+    State1.
 
 -spec install(rebar_state:t()) -> rebar_state:t().
 install(State) ->

src/rebar_prv_install_deps.erl

@@ -125,10 +125,7 @@ handle_deps_as_profile(Profile, State, Deps, Upgrade) ->
     DepsDir = profile_dep_dir(State, Profile),
     Deps1 = rebar_app_utils:parse_deps(DepsDir, Deps, State, Locks, Level),
     ProfileLevelDeps = [{Profile, Deps1, Level}],
-    Graph = rebar_state:packages_graph(State),
-    Registry = rebar_packages:registry(State),
-    State1 = rebar_state:packages_graph(rebar_state:registry(State, Registry), Graph),
-    handle_profile_level(ProfileLevelDeps, [], sets:new(), Upgrade, Locks, State1, Graph).
+    handle_profile_level(ProfileLevelDeps, [], sets:new(), Upgrade, Locks, State).
 
 %% ===================================================================
 %% Internal functions
@@ -141,10 +138,7 @@ deps_per_profile(Profiles, Upgrade, State) ->
     Deps = lists:foldl(fun(Profile, DepAcc) ->
                                [parsed_profile_deps(State, Profile, Level) | DepAcc]
                        end, [], Profiles),
-    Graph = rebar_state:packages_graph(State),
-    Registry = rebar_packages:registry(State),
-    State1 = rebar_state:packages_graph(rebar_state:registry(State, Registry), Graph),
-    handle_profile_level(Deps, [], sets:new(), Upgrade, Locks, State1, Graph).
+    handle_profile_level(Deps, [], sets:new(), Upgrade, Locks, State).
 
 parsed_profile_deps(State, Profile, Level) ->
     ParsedDeps = rebar_state:get(State, {parsed_deps, Profile}, []),
@@ -153,17 +147,17 @@ parsed_profile_deps(State, Profile, Level) ->
 %% Level-order traversal of all dependencies, across profiles.
 %% If profiles x,y,z are present, then the traversal will go:
 %% x0, y0, z0, x1, y1, z1, ..., xN, yN, zN.
-handle_profile_level([], Apps, _Seen, _Upgrade, _Locks, State, _Graph) ->
+handle_profile_level([], Apps, _Seen, _Upgrade, _Locks, State) ->
     {Apps, State};
-handle_profile_level([{Profile, Deps, Level} | Rest], Apps, Seen, Upgrade, Locks, State, Graph) ->
+handle_profile_level([{Profile, Deps, Level} | Rest], Apps, Seen, Upgrade, Locks, State) ->
     {Deps1, Apps1, State1, Seen1} =
         update_deps(Profile, Level, Deps, Apps
-                   ,State, Upgrade, Seen, Locks, Graph),
+                   ,State, Upgrade, Seen, Locks),
     Deps2 = case Deps1 of
         [] -> Rest;
         _ -> Rest ++ [{Profile, Deps1, Level+1}]
     end,
-    handle_profile_level(Deps2, Apps1, sets:union(Seen, Seen1), Upgrade, Locks, State1, Graph).
+    handle_profile_level(Deps2, Apps1, sets:union(Seen, Seen1), Upgrade, Locks, State1).
 
 find_cycles(Apps) ->
     case rebar_digraph:compile_order(Apps) of
@@ -201,17 +195,17 @@ maybe_lock(Profile, AppInfo, Seen, State, Level) ->
             {sets:add_element(Name, Seen), State}
     end.
 
-update_deps(Profile, Level, Deps, Apps, State, Upgrade, Seen, Locks, Graph) ->
+update_deps(Profile, Level, Deps, Apps, State, Upgrade, Seen, Locks) ->
     lists:foldl(
       fun(AppInfo, {DepsAcc, AppsAcc, StateAcc, SeenAcc}) ->
               update_dep(AppInfo, Profile, Level,
                          DepsAcc, AppsAcc, StateAcc,
-                         Upgrade, SeenAcc, Locks, Graph)
+                         Upgrade, SeenAcc, Locks)
       end,
       {[], Apps, State, Seen},
       rebar_utils:sort_deps(Deps)).
 
-update_dep(AppInfo, Profile, Level, Deps, Apps, State, Upgrade, Seen, Locks, Graph) ->
+update_dep(AppInfo, Profile, Level, Deps, Apps, State, Upgrade, Seen, Locks) ->
     %% If not seen, add to list of locks to write out
     Name = rebar_app_info:name(AppInfo),
     case sets:is_element(Name, Seen) of
@@ -222,7 +216,7 @@ update_dep(AppInfo, Profile, Level, Deps, Apps, State, Upgrade, Seen, Locks, Gra
         false ->
             update_unseen_dep(AppInfo, Profile, Level,
                               Deps, Apps,
-                              State, Upgrade, Seen, Locks, Graph)
+                              State, Upgrade, Seen, Locks)
     end.
 
 profile_dep_dir(State, Profile) ->
@@ -242,24 +236,23 @@ update_seen_dep(AppInfo, _Profile, _Level, Deps, Apps, State, Upgrade, Seen, Loc
     end,
     {Deps, Apps, State, Seen}.
 
-update_unseen_dep(AppInfo, Profile, Level, Deps, Apps, State, Upgrade, Seen, Locks, Graph) ->
+update_unseen_dep(AppInfo, Profile, Level, Deps, Apps, State, Upgrade, Seen, Locks) ->
     {NewSeen, State1} = maybe_lock(Profile, AppInfo, Seen, State, Level),
     {_, AppInfo1} = maybe_fetch(AppInfo, Profile, Upgrade, Seen, State1),
     DepsDir = profile_dep_dir(State, Profile),
     {AppInfo2, NewDeps, State2} =
-        handle_dep(State1, Profile, DepsDir, AppInfo1, Locks, Level, Graph),
+        handle_dep(State1, Profile, DepsDir, AppInfo1, Locks, Level),
     AppInfo3 = rebar_app_info:dep_level(AppInfo2, Level),
     {NewDeps ++ Deps, [AppInfo3 | Apps], State2, NewSeen}.
 
--spec handle_dep(rebar_state:t(), atom(), file:filename_all(), rebar_app_info:t(), list(), integer(), rebar_dict()) -> {rebar_app_info:t(), [rebar_app_info:t()], [pkg_dep()], [integer()], rebar_state:t()}.
-handle_dep(State, Profile, DepsDir, AppInfo, Locks, Level, Graph) ->
+-spec handle_dep(rebar_state:t(), atom(), file:filename_all(), rebar_app_info:t(), list(), integer()) -> {rebar_app_info:t(), [rebar_app_info:t()], [pkg_dep()], [integer()], rebar_state:t()}.
+handle_dep(State, Profile, DepsDir, AppInfo, Locks, Level) ->
     Profiles = rebar_state:current_profiles(State),
     Name = rebar_app_info:name(AppInfo),
     Vsn = rebar_app_info:original_vsn(AppInfo),
 
     %% Deps may be under a sub project app, find it and use its state if so
-    S0 = rebar_app_info:state(AppInfo),
-    S = rebar_state:registry(S0, rebar_state:registry(State)),
+    S = rebar_app_info:state(AppInfo),
     C = rebar_config:consult(rebar_app_info:dir(AppInfo)),
     S1 = rebar_state:new(S, C, rebar_app_info:dir(AppInfo)),
     S2 = rebar_state:apply_overrides(S1, Name),
@@ -278,7 +271,7 @@ handle_dep(State, Profile, DepsDir, AppInfo, Locks, Level, Graph) ->
     %% Upgrade lock level to be the level the dep will have in this dep tree
     case rebar_app_info:resource_type(AppInfo1) of
         pkg ->
-            NewDeps = digraph:out_neighbours(Graph, {ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)}),
+            NewDeps = rebar_packages:deps(Name, Vsn, S5),
             NewDeps1 = rebar_app_utils:parse_deps(Name, DepsDir, NewDeps, S5, Locks, Level+1),
             {rebar_app_info:deps(AppInfo1, NewDeps), NewDeps1, State};
         _ ->

src/rebar_prv_packages.erl

@@ -27,19 +27,17 @@ init(State) ->
 
 -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}.
 do(State) ->
-    Dict = rebar_packages:packages(State),
-    print_packages(Dict),
+    print_packages(),
     {ok, State}.
 
 -spec format_error(any()) -> iolist().
 format_error(load_registry_fail) ->
     "Failed to load package regsitry. Try running 'rebar3 update' to fix".
 
-print_packages(Dict) ->
-    Pkgs = lists:keysort(1, dict:fetch_keys(Dict)),
-    SortedPkgs = lists:foldl(fun({Pkg, Vsn}, Acc) ->
-                                       orddict:append_list(Pkg, [Vsn], Acc)
-                               end, orddict:new(), Pkgs),
+print_packages() ->
+    SortedPkgs = ets:foldl(fun({{Pkg, Vsn}, _}, Acc) ->
+                                   orddict:append_list(Pkg, [Vsn], Acc)
+                           end, orddict:new(), ?PACKAGE_TABLE),
 
     orddict:map(fun(Name, Vsns) ->
                         SortedVsns = lists:sort(fun(A, B) ->

src/rebar_prv_update.erl

@@ -9,21 +9,14 @@
          do/1,
          format_error/1]).
 
+-export([hex_to_index/1]).
+
 -include("rebar.hrl").
 -include_lib("providers/include/providers.hrl").
 
 -define(PROVIDER, update).
 -define(DEPS, []).
 
-%% Ignore warning of digraph opaque type when running dialyzer
--dialyzer({no_opaque, do/1}).
--dialyzer({no_opaque, write_registry/3}).
-
-%% Ignoring the opaque type warning won't stop dialyzer from warning of
-%% no return for functions that had the opaque type warnings
--dialyzer({no_return, do/1}).
--dialyzer({no_return, write_registry/3}).
-
 %% ===================================================================
 %% Public API
 %% ===================================================================
@@ -50,16 +43,15 @@ do(State) ->
         TmpDir = ec_file:insecure_mkdtemp(),
         TmpFile = filename:join(TmpDir, "packages.gz"),
 
-        Url = rebar_state:get(State, rebar_packages_cdn, "https://s3.amazonaws.com/s3.hex.pm/registry.ets.gz"),
+        Url = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_HEX_REGISTRY),
         {ok, _RequestId} = httpc:request(get, {Url, []},
                                          [], [{stream, TmpFile}, {sync, true}],
                                          rebar),
         {ok, Data} = file:read_file(TmpFile),
         Unzipped = zlib:gunzip(Data),
         ok = file:write_file(HexFile, Unzipped),
-        {Dict, Graph} = hex_to_graph(HexFile),
-        write_registry(Dict, Graph, State),
-        true = digraph:delete(Graph),
+
+        hex_to_index(State),
         ok
     catch
         _E:C ->
@@ -73,57 +65,58 @@ do(State) ->
 format_error(package_index_write) ->
     "Failed to write package index.".
 
--spec write_registry(rebar_dict(), {digraph, ets:tid(), ets:tid(), ets:tid(), any()}, rebar_state:t()) -> ok | {error, atom()}.
-write_registry(Dict, {digraph, Edges, Vertices, Neighbors, _}, State) ->
-    RegistryDir = rebar_packages:package_dir(State),
-    filelib:ensure_dir(filename:join(RegistryDir, "dummy")),
-    ets:tab2file(Edges, filename:join(RegistryDir, "edges")),
-    ets:tab2file(Vertices, filename:join(RegistryDir, "vertices")),
-    ets:tab2file(Neighbors, filename:join(RegistryDir, "neighbors")),
-    file:write_file(filename:join(RegistryDir, "dict"), term_to_binary(Dict)).
-
 is_supported(<<"make">>) -> true;
 is_supported(<<"rebar">>) -> true;
 is_supported(<<"rebar3">>) -> true;
 is_supported(_) -> false.
 
-hex_to_graph(Filename) ->
-    {ok, T} = ets:file2tab(Filename),
-    Graph = digraph:new(),
-    ets:foldl(fun({Pkg, [Versions]}, ok) when is_binary(Pkg), is_list(Versions) ->
-                      lists:foreach(fun(Version) ->
-                                            digraph:add_vertex(Graph, {Pkg, Version}, 1)
-                                    end, Versions);
-                 (_, ok) ->
-                      ok
-              end, ok, T),
-
-    Dict1 = ets:foldl(fun({{Pkg, PkgVsn}, [Deps, _, BuildTools | _]}, Dict) when is_list(BuildTools) ->
-                              case lists:any(fun is_supported/1, BuildTools) of
-                                  true ->
-                                      DepsList = update_graph(Pkg, PkgVsn, Deps, T, Graph),
-                                      dict:store({Pkg, PkgVsn}, DepsList, Dict);
-                                  false ->
-                                      Dict
-                              end;
-                         (_, Dict) ->
-                              Dict
-                      end, dict:new(), T),
-    {Dict1, Graph}.
-
-update_graph(Pkg, PkgVsn, Deps, HexRegistry, Graph) ->
+hex_to_index(State) ->
+    RegistryDir = rebar_packages:package_dir(State),
+    HexFile = filename:join(RegistryDir, "registry"),
+    try ets:file2tab(HexFile) of
+        {ok, Registry} ->
+            try
+                (catch ets:delete(?PACKAGE_TABLE)),
+                ets:new(?PACKAGE_TABLE, [named_table, public]),
+                ets:foldl(fun({{Pkg, PkgVsn}, [Deps, Checksum, BuildTools | _]}, _) when is_list(BuildTools) ->
+                                  case lists:any(fun is_supported/1, BuildTools) of
+                                      true ->
+                                          DepsList = update_deps_list(Deps, Registry, State),
+                                          ets:insert(?PACKAGE_TABLE, {{Pkg, PkgVsn}, DepsList, Checksum});
+                                      false ->
+                                          true
+                                  end;
+                             ({Pkg, [Vsns]}, _) when is_binary(Pkg) ->
+                                  ets:insert(?PACKAGE_TABLE, {Pkg, Vsns});
+                             (_, _) ->
+                                  true
+                          end, true, Registry),
+
+                ets:insert(?PACKAGE_TABLE, {package_index_version, ?PACKAGE_INDEX_VERSION}),
+                ets:tab2file(?PACKAGE_TABLE, filename:join(RegistryDir, "packages.idx")),
+                true
+            after
+                catch ets:delete(Registry)
+            end;
+        {error, Reason} ->
+            ?DEBUG("Error loading package registry: ~p", [Reason]),
+            false
+    catch
+        _:_ ->
+            fail
+    end.
+
+update_deps_list(Deps, HexRegistry, State) ->
     lists:foldl(fun([Dep, DepVsn, false, _AppName | _], DepsListAcc) ->
                         case DepVsn of
                             <<"~> ", Vsn/binary>> ->
-                                case rebar_packages:find_highest_matching(Dep, Vsn, HexRegistry) of
+                                case rebar_packages:find_highest_matching(Dep, Vsn, HexRegistry, State) of
                                     {ok, HighestDepVsn} ->
-                                        digraph:add_edge(Graph, {Pkg, PkgVsn}, {Dep, HighestDepVsn}),
-                                        [{Dep, DepVsn} | DepsListAcc];
+                                        [{Dep, HighestDepVsn} | DepsListAcc];
                                     none ->
                                         DepsListAcc
                                 end;
                             Vsn ->
-                                digraph:add_edge(Graph, {Pkg, PkgVsn}, {Dep, Vsn}),
                                 [{Dep, Vsn} | DepsListAcc]
                         end;
                    ([_Dep, _DepVsn, true, _AppName | _], DepsListAcc) ->

src/rebar_state.erl

@@ -38,10 +38,6 @@
          overrides/1, overrides/2,
          apply_overrides/2,
 
-         packages_graph/1, packages_graph/2,
-         packages/1,
-         registry/1, registry/2,
-
          resources/1, resources/2, add_resource/2,
          providers/1, providers/2, add_provider/2]).
 
@@ -66,9 +62,6 @@
                   all_plugin_deps     = []          :: [rebar_app_info:t()],
                   all_deps            = []          :: [rebar_app_info:t()],
 
-                  packages            = undefined   :: rebar_dict(),
-                  packages_graph      = undefined   :: rebar_digraph() | undefined,
-                  registry            = undefined   :: {ok, ets:tid()} | error | undefined,
                   overrides           = [],
                   resources           = [],
                   providers           = []}).
@@ -90,18 +83,9 @@ new(Config) when is_list(Config) ->
     Terms = [{{deps, default}, Deps}, {{plugins, default}, Plugins} | Config],
     true = rebar_config:verify_config_format(Terms),
     Opts = dict:from_list(Terms),
-    load_package_registry(
-      BaseState#state_t { dir = rebar_dir:get_cwd(),
-                          default = Opts,
-                          opts = Opts }).
-
-load_package_registry(Config0) ->
-    Registry = rebar_packages:registry(Config0),
-    Packages = rebar_packages:packages(Config0),
-    PackagesGraph = rebar_packages:packages_graph(Config0),
-    Config0#state_t{registry = Registry,
-                    packages_graph = PackagesGraph,
-                    packages = Packages}.
+    BaseState#state_t { dir = rebar_dir:get_cwd(),
+                        default = Opts,
+                        opts = Opts }.
 
 -spec new(t() | atom(), list()) -> t().
 new(Profile, Config) when is_atom(Profile)
@@ -113,11 +97,10 @@ new(Profile, Config) when is_atom(Profile)
     Terms = [{{deps, default}, Deps}, {{plugins, default}, Plugins} | Config],
     true = rebar_config:verify_config_format(Terms),
     Opts = dict:from_list(Terms),
-    load_package_registry(
-      BaseState#state_t { dir = rebar_dir:get_cwd(),
-                          current_profiles = [Profile],
-                          default = Opts,
-                          opts = Opts });
+    BaseState#state_t { dir = rebar_dir:get_cwd(),
+                        current_profiles = [Profile],
+                        default = Opts,
+                        opts = Opts };
 new(ParentState=#state_t{}, Config) ->
     %% Load terms from rebar.config, if it exists
     Dir = rebar_dir:get_cwd(),
@@ -450,27 +433,6 @@ namespace(#state_t{namespace=Namespace}) ->
 namespace(State=#state_t{}, Namespace) ->
     State#state_t{namespace=Namespace}.
 
-packages(#state_t{packages=undefined}) ->
-    throw(packages_usage_error);
-packages(#state_t{packages=Packages}) ->
-    Packages.
-
-packages_graph(#state_t{packages_graph=undefined}) ->
-    throw(packages_usage_error);
-packages_graph(#state_t{packages_graph=PackagesGraph}) ->
-    PackagesGraph.
-
-packages_graph(State, PackagesGraph) ->
-    State#state_t{packages_graph=PackagesGraph}.
-
-registry(#state_t{registry=undefined}) ->
-    throw(registry_usage_error);
-registry(#state_t{registry=Registry}) ->
-    Registry.
-
-registry(State, Registry) ->
-    State#state_t{registry=Registry}.
-
 -spec resources(t()) -> [{rebar_resource:type(), module()}].
 resources(#state_t{resources=Resources}) ->
     Resources.

test/mock_pkg_resource.erl

@@ -35,7 +35,7 @@ mock(Opts) ->
 
 unmock() ->
     meck:unload(?MOD),
-    meck:unload(rebar_state).
+    meck:unload(rebar_packages).
 
 %%%%%%%%%%%%%%%
 %%% Private %%%
@@ -111,34 +111,19 @@ mock_pkg_index(Opts) ->
     Deps = proplists:get_value(pkgdeps, Opts, []),
     Skip = proplists:get_value(not_in_index, Opts, []),
     %% Dict: {App, Vsn}: [{<<"link">>, <<>>}, {<<"deps">>, []}]
-    %% Digraph: all apps and deps in the index
+    %% Index: all apps and deps in the index
+
     Dict = find_parts(Deps, Skip),
-    GraphParts = to_graph_parts(Dict),
-    Digraph = rebar_digraph:restore_graph(GraphParts),
-    meck:new(rebar_state, [passthrough, no_link]),
-    meck:expect(rebar_state, registry,
-                fun(_State) -> {ok, to_registry(Deps)} end),
-    meck:expect(rebar_state, packages,
-                fun(_State) -> Dict end),
-    meck:expect(rebar_state, packages_graph,
-                fun(_State) -> Digraph end).
+    meck:new(rebar_packages, [passthrough, no_link]),
+    meck:expect(rebar_packages, packages,
+                fun(_State) -> to_index(Deps, Dict) end),
+    meck:expect(rebar_packages, load_and_verify_version,
+                fun(_State) -> to_index(Deps, Dict), true end).
 
 %%%%%%%%%%%%%%%
 %%% Helpers %%%
 %%%%%%%%%%%%%%%
 
-to_registry(Deps) ->
-    Tid = ets:new(registry, []),
-    lists:foreach(fun({{Name, Vsn}, _}) ->
-                          case ets:lookup(Tid, Name) of
-                              [{_, [Vsns]}] ->
-                                  ets:insert(Tid, {Name, [[Vsn | Vsns]]});
-                              _ ->
-                                  ets:insert(Tid, {Name, [[Vsn]]})
-                          end
-                  end, Deps),
-    Tid.
-
 all_files(Dir) ->
     filelib:wildcard(filename:join([Dir, "**"])).
 
@@ -158,10 +143,20 @@ find_parts([{AppName, Deps}|Rest], Skip, Acc) ->
             find_parts(Rest, Skip, AccNew)
     end.
 
-to_graph_parts(Dict) ->
-    LastUpdated = os:timestamp(),
-    dict:fold(fun(K,Deps,{Ks,Vs}) ->
-            {[{K,LastUpdated}|Ks],
-             [{K,{list_to_binary(atom_to_list(DK)), list_to_binary(DV)}}
-             || {DK,DV} <- Deps]  ++ Vs}
-        end, {[],[]}, Dict).
+to_index(AllDeps, Dict) ->
+    catch ets:delete(package_index),
+    ets:new(package_index, [named_table, public]),
+    dict:fold(
+      fun(K, Deps, _) ->
+              DepsList = [{ec_cnv:to_binary(DK), ec_cnv:to_binary(DV)} || {DK, DV} <- Deps],
+              ets:insert(package_index, {K, DepsList, <<"checksum">>})
+      end, ok, Dict),
+    ets:insert(package_index, {package_index_version, 3}),
+    lists:foreach(fun({{Name, Vsn}, _}) ->
+                          case ets:lookup(package_index,  ec_cnv:to_binary(Name)) of
+                              [{_, Vsns}] ->
+                                  ets:insert(package_index, {ec_cnv:to_binary(Name), [ec_cnv:to_binary(Vsn) | Vsns]});
+                              _ ->
+                                  ets:insert(package_index, {ec_cnv:to_binary(Name), [ec_cnv:to_binary(Vsn)]})
+                          end
+                  end, AllDeps).

test/rebar_pkg_SUITE.erl

@@ -159,25 +159,33 @@ mock_config(Name, Config) ->
     Priv = ?config(priv_dir, Config),
     CacheRoot = filename:join([Priv, "cache", atom_to_list(Name)]),
     TmpDir = filename:join([Priv, "tmp", atom_to_list(Name)]),
-    T = ets:new(fake_registry, [public]),
-    ets:insert_new(T, [
-        {{<<"badindexchk">>,<<"1.0.0">>}, [[], ?bad_checksum]},
-        {{<<"goodpkg">>,<<"1.0.0">>}, [[], ?good_checksum]},
-        {{<<"badpkg">>,<<"1.0.0">>}, [[], ?good_checksum]}
+    Tid = ets:new(registry_table, [public]),
+    ets:insert_new(Tid, [
+        {<<"badindexchk">>,[[<<"1.0.0">>]]},
+        {<<"goodpkg">>,[[<<"1.0.0">>]]},
+        {<<"badpkg">>,[[<<"1.0.0">>]]},
+        {{<<"badindexchk">>,<<"1.0.0">>}, [[], ?bad_checksum, [<<"rebar3">>]]},
+        {{<<"goodpkg">>,<<"1.0.0">>}, [[], ?good_checksum, [<<"rebar3">>]]},
+        {{<<"badpkg">>,<<"1.0.0">>}, [[], ?good_checksum, [<<"rebar3">>]]}
     ]),
     CacheDir = filename:join([CacheRoot, "hex", "com", "test", "packages"]),
     filelib:ensure_dir(filename:join([CacheDir, "registry"])),
-    ok = ets:tab2file(T, filename:join([CacheDir, "registry"])),
+    ok = ets:tab2file(Tid, filename:join([CacheDir, "registry"])),
+
     %% The state returns us a fake registry
     meck:new(rebar_state, [passthrough]),
-    meck:expect(rebar_state, registry,
-                fun(_State) -> {ok, T} end),
     meck:expect(rebar_state, get,
                 fun(_State, rebar_packages_cdn, _Default) ->
                     "http://test.com/"
                 end),
+
     meck:new(rebar_dir, [passthrough]),
     meck:expect(rebar_dir, global_cache_dir, fun(_) -> CacheRoot end),
+
+    meck:new(rebar_packages, [passthrough]),
+    meck:expect(rebar_packages, package_dir, fun(_) -> CacheDir end),
+    rebar_prv_update:hex_to_index(rebar_state:new()),
+
     %% Cache fetches are mocked -- we assume the server and clients are
     %% correctly used.
     GoodCache = ?config(good_cache, Config),
@@ -194,7 +202,7 @@ mock_config(Name, Config) ->
     [{cache_root, CacheRoot},
      {cache_dir, CacheDir},
      {tmp_dir, TmpDir},
-     {mock_table, T} | Config].
+     {mock_table, Tid} | Config].
 
 unmock_config(Config) ->
     meck:unload(),