Properly support package aliasing and alt names

Aliasing only had a bit of ad-hoc support in rebar3, and various issues
have encountered problems related to the package names not mapping
properly with the application name. One such issue is
https://github.com/erlang/rebar3/issues/1290

The problem has been hard to find because it only impacts transitive
dependencies (not top-level ones) of other packages. The root cause for
this is that the application name was not being tracked by rebar3's internal
index, only the package name and its version were.

When a given application was a package app, the data for the application
name would be reconstructed from the lock file, but only if it were a
top-level app or a dependency of a source application where parsing the
lock file is necessary to know what comes next.

When a transitive dependency of a package dependency was fetched, we
instead read its dependencies directly from the in-memory package index
within rebar3. This caused us to only read the package name and version,
and lost all information regarding application name. This worked fine
for most cases since for the vast majority of packages, the package name
matches the app name, but failed for all aliases, which would then be
moved to directories that wouldn't match the app name.

This in turn broke some aspects of code analysis (in Dialyzer), or other
functionality relying on static paths, such as including .hrl files from
dependencies.

This patch reformats the internal storage format of dependencies to
align with the internal one used by rebar3, so that the app name can be
carried along with the package name and its version.

The fix can only work once `rebar3 update` is called so the index is
rebuilt internally, and will the file cached on disk will be
incompatible with older rebar3 versions.

Currently, the following is not covered:

 - Tests
 - Including the package hashes of dependencies so they may match what
   is in a lock file -- they're being `undefined` instead, which may
   break some lookups. The previous format did not lend itself to hashing
   in the same way, and it is possible transitive deps were not being
   tracked properly, or worked by respecting the current package hierarchy.
   This will require further analysis

For now this commit can allow reviewing and discussion.

src/rebar_prv_update.erl

@@ -142,8 +142,8 @@ hex_to_index(State) ->
     end.
 
 update_deps_list(Pkg, PkgVsn, Deps, HexRegistry, State) ->
-    lists:foldl(fun([Dep, DepVsn, false, _AppName | _], DepsListAcc) ->
-                        Dep1 = {Pkg, PkgVsn, Dep},
+    lists:foldl(fun([Dep, DepVsn, false, AppName | _], DepsListAcc) ->
+                        Dep1 = {Pkg, PkgVsn, Dep, AppName},
                         case {valid_vsn(DepVsn), DepVsn} of
                             %% Those are all not perfectly implemented!
                             %% and doubled since spaces seem not to be
@@ -168,9 +168,9 @@ update_deps_list(Pkg, PkgVsn, Deps, HexRegistry, State) ->
                                 cmpl(Dep1, rm_ws(Vsn), HexRegistry, State,
                                      DepsListAcc, fun ec_semver:lt/2);
                             {_, <<"==", Vsn/binary>>} ->
-                                [{Dep, Vsn} | DepsListAcc];
+                                [{AppName, {pkg, Dep, Vsn, undefined}} | DepsListAcc];
                             {_, Vsn} ->
-                                [{Dep, Vsn} | DepsListAcc]
+                                [{AppName, {pkg, Dep, Vsn, undefined}} | DepsListAcc]
                         end;
                    ([_Dep, _DepVsn, true, _AppName | _], DepsListAcc) ->
                         DepsListAcc
@@ -188,27 +188,27 @@ valid_vsn(Vsn) ->
     SupportedVersions = "^(>=?|<=?|~>|==)?\\s*" ++ SemVerRegExp ++ "$",
     re:run(Vsn, SupportedVersions) =/= nomatch.
 
-highest_matching({Pkg, PkgVsn, Dep}, Vsn, HexRegistry, State, DepsListAcc) ->
+highest_matching({Pkg, PkgVsn, Dep, App}, Vsn, HexRegistry, State, DepsListAcc) ->
     case rebar_packages:find_highest_matching(Pkg, PkgVsn, Dep, Vsn, HexRegistry, State) of
         {ok, HighestDepVsn} ->
-            [{Dep, HighestDepVsn} | DepsListAcc];
+            [{App, {pkg, Dep, HighestDepVsn, undefined}} | DepsListAcc];
         none ->
             ?WARN("[~s:~s] Missing registry entry for package ~s. Try to fix with `rebar3 update`",
                   [Pkg, PkgVsn, Dep]),
             DepsListAcc
     end.
 
-cmp({_Pkg, _PkgVsn, Dep} = Dep1, Vsn, HexRegistry, State, DepsListAcc, CmpFun) ->
+cmp({_Pkg, _PkgVsn, Dep, _App} = Dep1, Vsn, HexRegistry, State, DepsListAcc, CmpFun) ->
     {ok, Vsns}  = rebar_packages:find_all(Dep, HexRegistry, State),
     cmp_(undefined, Vsn, Vsns, DepsListAcc, Dep1, CmpFun).
 
 
-cmp_(undefined, _MinVsn, [], DepsListAcc, {Pkg, PkgVsn, Dep}, _CmpFun) ->
+cmp_(undefined, _MinVsn, [], DepsListAcc, {Pkg, PkgVsn, Dep, _App}, _CmpFun) ->
     ?WARN("[~s:~s] Missing registry entry for package ~s. Try to fix with `rebar3 update`",
           [Pkg, PkgVsn, Dep]),
     DepsListAcc;
-cmp_(HighestDepVsn, _MinVsn, [], DepsListAcc, {_Pkg, _PkgVsn, Dep}, _CmpFun) ->
-    [{Dep, HighestDepVsn} | DepsListAcc];
+cmp_(HighestDepVsn, _MinVsn, [], DepsListAcc, {_Pkg, _PkgVsn, Dep, App}, _CmpFun) ->
+    [{App, {pkg, Dep, HighestDepVsn, undefined}} | DepsListAcc];
 
 cmp_(BestMatch, MinVsn, [Vsn | R], DepsListAcc, Dep, CmpFun) ->
     case CmpFun(Vsn, MinVsn) of
@@ -224,13 +224,13 @@ cmpl({_Pkg, _PkgVsn, Dep} = Dep1, Vsn, HexRegistry, State, DepsListAcc, CmpFun)
     {ok, Vsns}  = rebar_packages:find_all(Dep, HexRegistry, State),
     cmpl_(undefined, Vsn, Vsns, DepsListAcc, Dep1, CmpFun).
 
-cmpl_(undefined, _MaxVsn, [], DepsListAcc, {Pkg, PkgVsn, Dep}, _CmpFun) ->
+cmpl_(undefined, _MaxVsn, [], DepsListAcc, {Pkg, PkgVsn, Dep, _App}, _CmpFun) ->
     ?WARN("[~s:~s] Missing registry entry for package ~s. Try to fix with `rebar3 update`",
           [Pkg, PkgVsn, Dep]),
     DepsListAcc;
 
-cmpl_(HighestDepVsn, _MaxVsn, [], DepsListAcc, {_Pkg, _PkgVsn, Dep}, _CmpFun) ->
-    [{Dep, HighestDepVsn} | DepsListAcc];
+cmpl_(HighestDepVsn, _MaxVsn, [], DepsListAcc, {_Pkg, _PkgVsn, Dep, App}, _CmpFun) ->
+    [{App, {pkg, Dep, HighestDepVsn, undefined}} | DepsListAcc];
 
 cmpl_(undefined, MaxVsn, [Vsn | R], DepsListAcc, Dep, CmpFun) ->
     case CmpFun(Vsn, MaxVsn) of

test/rebar_deps_SUITE.erl

@@ -385,36 +385,36 @@ https_os_proxy_settings(_Config) ->
 
 semver_matching_lt(_Config) ->
     Dep = <<"test">>,
-    Dep1 = {Dep, <<"1.0.0">>, Dep},
+    Dep1 = {Dep, <<"1.0.0">>, Dep, Dep},
     MaxVsn = <<"0.2.0">>,
     Vsns = [<<"0.1.7">>, <<"0.1.9">>, <<"0.1.8">>, <<"0.2.0">>, <<"0.2.1">>],
-    ?assertEqual([{Dep, <<"0.1.9">>}],
+    ?assertEqual([{Dep, {pkg, Dep, <<"0.1.9">>, undefined}}],
                  rebar_prv_update:cmpl_(undefined, MaxVsn, Vsns, [], Dep1,
                                         fun ec_semver:lt/2)).
 
 semver_matching_lte(_Config) ->
     Dep = <<"test">>,
-    Dep1 = {Dep, <<"1.0.0">>, Dep},
+    Dep1 = {Dep, <<"1.0.0">>, Dep, Dep},
     MaxVsn = <<"0.2.0">>,
     Vsns = [<<"0.1.7">>, <<"0.1.9">>, <<"0.1.8">>, <<"0.2.0">>, <<"0.2.1">>],
-    ?assertEqual([{Dep, <<"0.2.0">>}],
+    ?assertEqual([{Dep, {pkg, Dep, <<"0.2.0">>, undefined}}],
                  rebar_prv_update:cmpl_(undefined, MaxVsn, Vsns, [], Dep1,
                                         fun ec_semver:lte/2)).
 
 semver_matching_gt(_Config) ->
     Dep = <<"test">>,
-    Dep1 = {Dep, <<"1.0.0">>, Dep},
+    Dep1 = {Dep, <<"1.0.0">>, Dep, Dep},
     MaxVsn = <<"0.2.0">>,
     Vsns = [<<"0.1.7">>, <<"0.1.9">>, <<"0.1.8">>, <<"0.2.0">>, <<"0.2.1">>],
-    ?assertEqual([{Dep, <<"0.2.1">>}],
+    ?assertEqual([{Dep, {pkg, Dep, <<"0.2.1">>, undefined}}],
                  rebar_prv_update:cmp_(undefined, MaxVsn, Vsns, [], Dep1,
                                        fun ec_semver:gt/2)).
 semver_matching_gte(_Config) ->
     Dep = <<"test">>,
-    Dep1 = {Dep, <<"1.0.0">>, Dep},
+    Dep1 = {Dep, <<"1.0.0">>, Dep, Dep},
     MaxVsn = <<"0.2.0">>,
     Vsns = [<<"0.1.7">>, <<"0.1.9">>, <<"0.1.8">>, <<"0.2.0">>],
-    ?assertEqual([{Dep, <<"0.2.0">>}],
+    ?assertEqual([{Dep, {pkg, Dep, <<"0.2.0">>, undefined}}],
                  rebar_prv_update:cmp_(undefined, MaxVsn, Vsns, [], Dep1,
                                        fun ec_semver:gt/2)).