use hex_core tarball unpacking support in pkg resource (#1883)

* use hex_core tarball unpacking support in pkg resource

* ignore etag if package doesn't exist and delete if checksum fails

* add back tests for bad package checksums

* improve bad registry checksum error message

src/rebar_fetch.erl

@@ -71,26 +71,10 @@ needs_update(AppInfo, State) ->
             true
     end.
 
-format_error({bad_download, CachePath}) ->
-    io_lib:format("Download of package does not match md5sum from server: ~ts", [CachePath]);
-format_error({unexpected_hash, CachePath, Expected, Found}) ->
-    io_lib:format("The checksum for package at ~ts (~ts) does not match the "
-                  "checksum previously locked (~ts). Either unlock or "
-                  "upgrade the package, or make sure you fetched it from "
-                  "the same index from which it was initially fetched.",
-                  [CachePath, Found, Expected]);
-format_error({failed_extract, CachePath}) ->
-    io_lib:format("Failed to extract package: ~ts", [CachePath]);
-format_error({bad_etag, Source}) ->
-    io_lib:format("MD5 Checksum comparison failed for: ~ts", [Source]);
 format_error({fetch_fail, Name, Vsn}) ->
     io_lib:format("Failed to fetch and copy dep: ~ts-~ts", [Name, Vsn]);
 format_error({fetch_fail, Source}) ->
     io_lib:format("Failed to fetch and copy dep: ~p", [Source]);
-format_error({bad_checksum, File}) ->
-    io_lib:format("Checksum mismatch against tarball in ~ts", [File]);
-format_error({bad_registry_checksum, File}) ->
-    io_lib:format("Checksum mismatch against registry in ~ts", [File]);
 format_error({dep_app_not_found, AppName}) ->
     io_lib:format("Dependency failure: source for ~ts does not contain a "
                   "recognizable project and can not be built", [AppName]).

src/rebar_pkg_resource.erl

@@ -9,20 +9,18 @@
          download/4,
          download/5,
          needs_update/2,
-         make_vsn/2]).
+         make_vsn/2,
+         format_error/1]).
 
 -ifdef(TEST).
 %% exported for test purposes
--export([store_etag_in_cache/2, etag/1, request/4]).
+-export([store_etag_in_cache/2]).
 -endif.
 
 -include("rebar.hrl").
+-include_lib("providers/include/providers.hrl").
 
--type cached_result()   :: ok |
-                           {bad_checksum,string()} |
-                           {bad_registry_checksum,string()} |
-                           {failed_extract,string()} |
-                           {unexpected_hash,string(),_,binary()}.
+-type package() :: {pkg, binary(), binary(), binary(), rebar_hex_repos:repo()}.
 
 %%==============================================================================
 %% Public API
@@ -97,19 +95,27 @@ download(TmpDir, AppInfo, State, ResourceState) ->
 %%------------------------------------------------------------------------------
 -spec download(TmpDir, Pkg, State, ResourceState, UpdateETag) -> Res when
       TmpDir :: file:name(),
-      Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary(), RepoConfig :: rebar_hex_repos:repo()},
+      Pkg :: package(),
       State :: rebar_state:t(),
       ResourceState:: rebar_resource_v2:resource_state(),
       UpdateETag :: boolean(),
-      Res :: ok | {error,_} | cached_result() | {fetch_fail, binary(), binary()} | {bad_download, file:name()}.
+      Res :: ok | {error,_} | {unexpected_hash, string(), integer(), integer()} |
+             {fetch_fail, binary(), binary()}.
 download(TmpDir, Pkg={pkg, Name, Vsn, _Hash, Repo}, State, _ResourceState, UpdateETag) ->
     {ok, PackageDir} = rebar_packages:package_dir(Repo, State),
     Package = binary_to_list(<<Name/binary, "-", Vsn/binary, ".tar">>),
     ETagFile = binary_to_list(<<Name/binary, "-", Vsn/binary, ".etag">>),
     CachePath = filename:join(PackageDir, Package),
     ETagPath = filename:join(PackageDir, ETagFile),
-    cached_download(TmpDir, CachePath, Pkg, etag(ETagPath),
-                    State, ETagPath, UpdateETag).
+    case cached_download(TmpDir, CachePath, Pkg, etag(CachePath, ETagPath), ETagPath, UpdateETag) of
+        {bad_registry_checksum, Expected, Found} ->
+            %% checksum comparison failed. in case this is from a modified cached package
+            %% overwrite the etag if it exists so it is not relied on again
+            store_etag_in_cache(ETagPath, <<>>),
+            ?PRV_ERROR({bad_registry_checksum, Name, Vsn, Expected, Found});
+        Result ->
+            Result
+    end.
 
 %%------------------------------------------------------------------------------
 %% @doc
@@ -124,6 +130,12 @@ download(TmpDir, Pkg={pkg, Name, Vsn, _Hash, Repo}, State, _ResourceState, Updat
 make_vsn(_, _) ->
     {error, "Replacing version of type pkg not supported."}.
 
+format_error({bad_registry_checksum, Name, Vsn, Expected, Found}) ->
+    io_lib:format("The checksum for package at ~ts-~ts (~ts) does not match the "
+                  "checksum expected from the registry (~ts). "
+                  "Run `rebar3 do unlock ~ts, update` and then try again.",
+                  [Name, Vsn, Found, Expected, Name]).
+
 %%------------------------------------------------------------------------------
 %% @doc
 %% Download the pkg belonging to the given address. If the etag of the pkg
@@ -160,13 +172,21 @@ request(Config, Name, Version, ETag) ->
 %% returned from the hexpm server. The name is package-vsn.etag.
 %% @end
 %%------------------------------------------------------------------------------
--spec etag(Path) -> Res when
-      Path :: file:name(),
+-spec etag(PackagePath, ETagPath) -> Res when
+      PackagePath :: file:name(),
+      ETagPath :: file:name(),
       Res :: binary().
-etag(Path) ->
-    case file:read_file(Path) of
+etag(PackagePath, ETagPath) ->
+    case file:read_file(ETagPath) of
         {ok, Bin} ->
-            Bin;
+            %% just in case a user deleted a cached package but not its etag
+            %% verify the package is also there, and if not, ignore the etag
+            case filelib:is_file(PackagePath) of
+                true ->
+                    Bin;
+                false ->
+                    <<>>
+            end;
         {error, _} ->
             <<>>
     end.
@@ -186,115 +206,69 @@ store_etag_in_cache(Path, ETag) ->
 %%%=============================================================================
 %%% Private functions
 %%%=============================================================================
--spec cached_download(TmpDir, CachePath, Pkg, ETag, State, ETagPath,
-                      UpdateETag) -> Res when
+-spec cached_download(TmpDir, CachePath, Pkg, ETag, ETagPath, UpdateETag) -> Res when
       TmpDir :: file:name(),
       CachePath :: file:name(),
-      Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary(), RepoConfig :: rebar_hex_repos:repo()},
+      Pkg :: package(),
       ETag :: binary(),
-      State :: rebar_state:t(),
       ETagPath :: file:name(),
       UpdateETag :: boolean(),
-      Res :: cached_result() | {fetch_fail, binary(), binary()} | {bad_download, file:name()}.
+      Res :: ok | {unexpected_hash, integer(), integer()} | {fetch_fail, binary(), binary()}.
 cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn, _Hash, RepoConfig}, ETag,
-                State, ETagPath, UpdateETag) ->
+                ETagPath, UpdateETag) ->
     case request(RepoConfig, Name, Vsn, ETag) of
         {ok, cached} ->
             ?INFO("Version cached at ~ts is up to date, reusing it", [CachePath]),
-            serve_from_cache(TmpDir, CachePath, Pkg, State);
+            serve_from_cache(TmpDir, CachePath, Pkg);
         {ok, Body, NewETag} ->
             ?INFO("Downloaded package, caching at ~ts", [CachePath]),
             maybe_store_etag_in_cache(UpdateETag, ETagPath, NewETag),
-            serve_from_download(TmpDir, CachePath, Pkg, NewETag, Body, 
-                                State, ETagPath);
+            serve_from_download(TmpDir, CachePath, Pkg, Body);
         error when ETag =/= <<>> ->
             store_etag_in_cache(ETagPath, ETag),
             ?INFO("Download error, using cached file at ~ts", [CachePath]),
-            serve_from_cache(TmpDir, CachePath, Pkg, State);
+            serve_from_cache(TmpDir, CachePath, Pkg);
         error ->
             {fetch_fail, Name, Vsn}
     end.
 
--spec serve_from_cache(TmpDir, CachePath, Pkg, State) -> Res when
+-spec serve_from_cache(TmpDir, CachePath, Pkg) -> Res when
       TmpDir :: file:name(),
       CachePath :: file:name(),
-      Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary(), RepoConfig :: rebar_hex_repos:repo()},
-      State :: rebar_state:t(),
-      Res :: cached_result().
-serve_from_cache(TmpDir, CachePath, Pkg, State) ->
-    {Files, Contents, Version, Meta} = extract(TmpDir, CachePath),
-    case checksums(Pkg, Files, Contents, Version, Meta, State) of
-        {Chk, Chk, Chk} ->
-            erl_tar:extract({binary, Contents}, [{cwd, TmpDir}, compressed]);
-        {_Hash, Chk, Chk} ->
-            ?DEBUG("Expected hash ~p does not match checksums ~p", [_Hash, Chk]),
-            {unexpected_hash, CachePath, _Hash, Chk};
-        {Chk, _Bin, Chk} ->
-            ?DEBUG("Checksums: registry: ~p, pkg: ~p", [Chk, _Bin]),
-            {failed_extract, CachePath};
-        {_Hash, _Bin, _Tar} ->
-            ?DEBUG("Checksums: expected: ~p, pkg: ~p, meta: ~p",
-                   [_Hash, _Bin, _Tar]),
-            {bad_checksum, CachePath}
+      Pkg :: package(),
+      Res :: ok | {error,_} | {bad_registry_checksum, integer(), integer()}.
+serve_from_cache(TmpDir, CachePath, Pkg) ->
+    {ok, Binary} = file:read_file(CachePath),
+    serve_from_memory(TmpDir, Binary, Pkg).
+
+-spec serve_from_memory(TmpDir, Tarball, Package) -> Res when
+      TmpDir :: file:name(),
+      Tarball :: binary(),
+      Package :: package(),
+      Res :: ok | {error,_} | {bad_registry_checksum, integer(), integer()}.
+serve_from_memory(TmpDir, Binary, {pkg, _Name, _Vsn, Hash, _RepoConfig}) ->
+    RegistryChecksum = list_to_integer(binary_to_list(Hash), 16),
+    case hex_tarball:unpack(Binary, TmpDir) of
+        {ok, #{checksum := <<Checksum:256/big-unsigned>>}} when RegistryChecksum =/= Checksum ->
+            ?DEBUG("Expected hash ~64.16.0B does not match checksum of fetched package ~64.16.0B",
+                   [RegistryChecksum, Checksum]),
+            {bad_registry_checksum, RegistryChecksum, Checksum};
+        {ok, #{checksum := <<RegistryChecksum:256/big-unsigned>>}} ->
+            ok;
+        {error, Reason} ->
+            {error, {hex_tarball, Reason}}
     end.
 
--spec serve_from_download(TmpDir, CachePath, Package, ETag, Binary, State,
-                          ETagPath) -> Res when
+-spec serve_from_download(TmpDir, CachePath, Package, Binary) -> Res when
       TmpDir :: file:name(),
       CachePath :: file:name(),
-      Package :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary(), RepoConfig :: rebar_hex_repos:repo()},
-      ETag :: binary(),
+      Package :: package(),
       Binary :: binary(),
-      State :: rebar_state:t(),
-      ETagPath :: file:name(),
-      Res :: ok | {error,_} | {bad_download, file:name()}.
-serve_from_download(TmpDir, CachePath, Package, ETag, Binary, State, ETagPath) ->
+      Res :: ok | {error,_}.
+serve_from_download(TmpDir, CachePath, Package, Binary) ->
     ?DEBUG("Writing ~p to cache at ~ts", [Package, CachePath]),
     file:write_file(CachePath, Binary),
-    case etag(ETagPath) of
-        ETag ->
-            serve_from_cache(TmpDir, CachePath, Package, State);
-        FileETag ->            
-            ?DEBUG("Downloaded file ~ts ETag ~ts doesn't match returned ETag ~ts",
-                   [CachePath, ETag, FileETag]),
-            {bad_download, CachePath}
-    end.
-
--spec extract(TmpDir, CachePath) -> Res when
-      TmpDir :: file:name(),
-      CachePath :: file:name(),
-      Res :: {Files, Contents, Version, Meta},
-      Files :: list({file:name(), binary()}),
-      Contents :: binary(),
-      Version :: binary(),
-      Meta :: binary().
-extract(TmpDir, CachePath) ->
-    ec_file:mkdir_p(TmpDir),
-    {ok, Files} = erl_tar:extract(CachePath, [memory]),
-    {"contents.tar.gz", Contents} = lists:keyfind("contents.tar.gz", 1, Files),
-    {"VERSION", Version} = lists:keyfind("VERSION", 1, Files),
-    {"metadata.config", Meta} = lists:keyfind("metadata.config", 1, Files),
-    {Files, Contents, Version, Meta}.
-
--spec checksums(Pkg, Files, Contents, Version, Meta, State) -> Res when
-      Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary(), RepoConfig :: rebar_hex_repos:repo()},
-      Files :: list({file:name(), binary()}),
-      Contents :: binary(),
-      Version :: binary(),
-      Meta :: binary(),
-      State :: rebar_state:t(),
-      Res :: {Hash, BinChecksum, TarChecksum},
-      Hash :: binary(),
-      BinChecksum :: binary(),
-      TarChecksum :: binary().
-checksums({pkg, _Name, _Vsn, Hash, _}, Files, Contents, Version, Meta, _State) ->
-    Blob = <<Version/binary, Meta/binary, Contents/binary>>,
-    <<X:256/big-unsigned>> = crypto:hash(sha256, Blob),
-    BinChecksum = list_to_binary(
-                    rebar_string:uppercase(
-                      lists:flatten(io_lib:format("~64.16.0b", [X])))),
-    {"CHECKSUM", TarChecksum} = lists:keyfind("CHECKSUM", 1, Files),
-    {Hash, BinChecksum, TarChecksum}.
+    serve_from_memory(TmpDir, Binary, Package).
 
 -spec maybe_store_etag_in_cache(UpdateETag, Path, ETag) -> Res when
       UpdateETag :: boolean(),

test/mock_pkg_resource.erl

@@ -93,20 +93,21 @@ mock_download(Opts) ->
                 [kernel, stdlib] ++ [element(1,D) || D  <- AppDeps]
             ),
             rebar_test_utils:create_config(Dir, [{deps, AppDeps}]++Config),
+
             TarApp = App++"-"++rebar_utils:to_list(Vsn)++".tar",
-            Tarball = filename:join([Dir, TarApp]),
-            Contents = filename:join([Dir, "contents.tar.gz"]),
+
+            Metadata = #{<<"app">> => AppBin,
+                         <<"version">> => Vsn},
+
             Files = all_files(rebar_app_info:dir(AppInfo1)),
-            ok = erl_tar:create(Contents,
-                                archive_names(Dir, App, Vsn, Files),
-                                [compressed]),
-            ok = erl_tar:create(Tarball,
-                                [{"contents.tar.gz", Contents}],
-                                []),
+            {ok, {Tarball, _Checksum}} = hex_tarball:create(Metadata, archive_names(Dir, Files)),
+            Archive = filename:join([Dir, TarApp]),
+            file:write_file(Archive, Tarball),
+
             Cache = proplists:get_value(cache_dir, Opts, filename:join(Dir,"cache")),
             Cached = filename:join([Cache, TarApp]),
             filelib:ensure_dir(Cached),
-            rebar_file_utils:mv(Tarball, Cached),
+            rebar_file_utils:mv(Archive, Cached),
             ok
         end).
 
@@ -137,7 +138,7 @@ mock_pkg_index(Opts) ->
 all_files(Dir) ->
     filelib:wildcard(filename:join([Dir, "**"])).
 
-archive_names(Dir, _App, _Vsn, Files) ->
+archive_names(Dir, Files) ->
     [{(F -- Dir) -- "/", F} || F <- Files].
 
 find_parts(Apps, Skip) -> find_parts(Apps, Skip, dict:new()).

test/rebar_pkg_SUITE.erl

@@ -8,14 +8,15 @@
 
 -define(bad_etag, <<"abcdef">>).
 -define(good_etag, <<"22e1d7387c9085a462340088a2a8ba67">>).
+-define(badpkg_checksum, <<"A14E3718B33F8124E98004433193509EC6660F6CA03302657CAB8785751D77A0">>).
+-define(badindex_checksum, <<"7B2CBED315C89F3126B5BF553DD7FF0FB5FE94B064888DD1B095CE8BF4B6A16A">>).
 -define(bad_checksum, <<"D576B442A68C7B92BACDE1EFE9C6E54D8D6C74BDB71D8175B9D3C6EC8C7B62A7">>).
--define(good_checksum, <<"1C6CE379D191FBAB41B7905075E0BF87CBBE23C77CECE775C5A0B786B2244C35">>).
+-define(good_checksum, <<"12726BDE1F65583A0817A7E8AADCA73F03FD8CB06F01E6CD29117C4A0DA0AFCF">>).
 -define(BADPKG_ETAG, <<"BADETAG">>).
 
-all() -> [good_uncached, good_cached, badpkg,
-          %% badindexchk, badhash_nocache, badhash_cache,
-          bad_to_good, good_disconnect, bad_disconnect, pkgs_provider,
-          find_highest_matching].
+all() -> [good_uncached, good_cached, badpkg, badhash_nocache,
+          badindexchk, badhash_cache, bad_to_good, good_disconnect,
+          bad_disconnect, pkgs_provider, find_highest_matching].
 
 init_per_suite(Config) ->
     application:start(meck),
@@ -96,9 +97,6 @@ init_per_testcase(bad_disconnect=Name, Config0) ->
                {pkg, Pkg}
               | Config0],
     Config = mock_config(Name, Config1),
-    %% meck:unload(httpc),
-    %% meck:new(httpc, [passthrough, unsticky]),
-    %% meck:expect(httpc, request, fun(_, _, _, _) -> {error, econnrefused} end),
     meck:expect(hex_repo, get_tarball, fun(_, _, _) ->
                                                {error, econnrefused}
                                        end),                
@@ -134,6 +132,17 @@ good_cached(Config) ->
                  rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum, #{}}, State, #{}, true)),
     {ok, Content} = file:read_file(CachedFile).
 
+
+badindexchk(Config) ->
+    Tmp = ?config(tmp_dir, Config),
+    {Pkg,Vsn} = ?config(pkg, Config),
+    State = ?config(state, Config),
+    ?assertMatch({error, {rebar_pkg_resource, {bad_registry_checksum, _, _, _, _}}},
+                 rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?bad_checksum, #{}}, State, #{}, true)),
+    %% The cached file is there for forensic purposes
+    Cache = ?config(cache_dir, Config),
+    ?assert(filelib:is_regular(filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".tar">>))).
+
 badpkg(Config) ->
     Tmp = ?config(tmp_dir, Config),
     {Pkg,Vsn} = ?config(pkg, Config),
@@ -142,12 +151,36 @@ badpkg(Config) ->
     CachePath = filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".tar">>),
     ETagPath = filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".etag">>),
     rebar_pkg_resource:store_etag_in_cache(ETagPath, ?BADPKG_ETAG),
-    ?assertMatch({bad_download, _Path},
-                 rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum, #{}}, State, #{}, false)),
+    ?assertMatch({error, {hex_tarball, {tarball, {checksum_mismatch, _, _}}}},
+                 rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?badpkg_checksum, #{}}, State, #{}, false)),
     %% The cached/etag files are there for forensic purposes
     ?assert(filelib:is_regular(ETagPath)),
     ?assert(filelib:is_regular(CachePath)).
 
+badhash_nocache(Config) ->
+    Tmp = ?config(tmp_dir, Config),
+    {Pkg,Vsn} = ?config(pkg, Config),
+    State = ?config(state, Config),
+    ?assertMatch({error, {rebar_pkg_resource, {bad_registry_checksum, _, _, _, _}}},
+                 rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?bad_checksum, #{}}, State, #{}, true)),
+    %% The cached file is there for forensic purposes
+    Cache = ?config(cache_dir, Config),
+    ?assert(filelib:is_regular(filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".tar">>))).
+
+badhash_cache(Config) ->
+    Tmp = ?config(tmp_dir, Config),
+    {Pkg,Vsn} = ?config(pkg, Config),
+    Cache = ?config(cache_dir, Config),
+    State = ?config(state, Config),
+    CachedFile = filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".tar">>),
+    ?assert(filelib:is_regular(CachedFile)),
+    {ok, Content} = file:read_file(CachedFile),
+    ?assertMatch({error, {rebar_pkg_resource, {bad_registry_checksum, _, _, _, _}}},
+                 rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?bad_checksum, #{}}, State, #{}, true)),
+    %% The cached file is there still, unchanged.
+    ?assert(filelib:is_regular(CachedFile)),
+    ?assertEqual({ok, Content}, file:read_file(CachedFile)).
+
 bad_to_good(Config) ->
     Tmp = ?config(tmp_dir, Config),
     {Pkg,Vsn} = ?config(pkg, Config),
@@ -220,7 +253,7 @@ mock_config(Name, Config) ->
         {{<<"goodpkg">>,<<"1.0.1">>}, [[], ?good_checksum, [<<"rebar3">>]]},
         {{<<"goodpkg">>,<<"1.1.1">>}, [[], ?good_checksum, [<<"rebar3">>]]},
         {{<<"goodpkg">>,<<"2.0.0">>}, [[], ?good_checksum, [<<"rebar3">>]]},
-        {{<<"badpkg">>,<<"1.0.0">>}, [[], ?good_checksum, [<<"rebar3">>]]}
+        {{<<"badpkg">>,<<"1.0.0">>}, [[], ?badpkg_checksum, [<<"rebar3">>]]}
     ],
     ets:insert_new(Tid, AllDeps),
     CacheDir = filename:join([CacheRoot, "hex", "com", "test", "packages"]),

test/rebar_pkg_alias_SUITE.erl

@@ -190,7 +190,7 @@ mock_config(Name, Config) ->
     CacheDir = filename:join([CacheRoot, "hex", "com", "test", "packages"]),
     rebar_test_utils:create_app(AppDir, "fakelib", "1.0.0", [kernel, stdlib]),
     ct:pal("{~p, ~p}",[ChkFake, Etag]),
-    {ChkFake, Etag} = rebar_test_utils:package_app(AppDir, CacheDir, "goodpkg-1.0.0"),
+    {ChkGood, EtagGood} = rebar_test_utils:package_app(AppDir, CacheDir, "goodpkg", "1.0.0"),
 
     AllDeps = [
      {<<"fakelib">>,[[<<"1.0.0">>]]},
@@ -198,7 +198,7 @@ mock_config(Name, Config) ->
      {<<"topdep">>,[[<<"1.0.0">>]]},
      {<<"transitive">>, [[<<"1.0.0">>]]},
      {{<<"fakelib">>,<<"1.0.0">>}, [[], ChkFake, [<<"rebar3">>]]},
-     {{<<"goodpkg">>,<<"1.0.0">>}, [[], ChkFake, [<<"rebar3">>]]},
+     {{<<"goodpkg">>,<<"1.0.0">>}, [[], ChkGood, [<<"rebar3">>]]},
      {{<<"topdep">>,<<"1.0.0">>},
       [[
         {<<"transitive">>, <<"1.0.0">>, false, <<"transitive_app">>}
@@ -248,13 +248,14 @@ mock_config(Name, Config) ->
                         Releases =
                             [#{checksum => Checksum,
                                version => Vsn,
-                               dependencies => [{DAppName, {pkg, DN, DV, undefined}} || {DN, DV, _, DAppName} <- Deps]} ||
+                               dependencies => [{DAppName, {pkg, DN, DV, undefined}} ||
+                                                   {DN, DV, _, DAppName} <- Deps]} ||
                                 {{_, Vsn}, [Deps, Checksum, _]} <- Matches],
                         {ok, {200, #{}, #{releases => Releases}}}
                 end),
 
     meck:expect(hex_repo, get_tarball, fun(_, _, _) ->
-                                               {ok, {304, #{<<"etag">> => Etag}, <<>>}}
+                                               {ok, {304, #{<<"etag">> => EtagGood}, <<>>}}
                                        end),   
     
     %% Move all packages to cache
@@ -278,4 +279,4 @@ create_lib(Name, Config, AppName, PkgName) ->
     CacheDir = filename:join([CacheRoot, "hex", "com", "test", "packages"]),
     filelib:ensure_dir(filename:join([CacheDir, "registry"])),
     rebar_test_utils:create_app(AppDir, AppName, "1.0.0", [kernel, stdlib]),
-    rebar_test_utils:package_app(AppDir, CacheDir, PkgName++"-1.0.0").
+    rebar_test_utils:package_app(AppDir, CacheDir, PkgName, "1.0.0").

test/rebar_test_utils.erl

@@ -4,7 +4,7 @@
 -export([init_rebar_state/1, init_rebar_state/2, run_and_check/4, check_results/3]).
 -export([expand_deps/2, flat_deps/1, top_level_deps/1]).
 -export([create_app/4, create_plugin/4, create_eunit_app/4, create_empty_app/4,
-         create_config/2, create_config/3, package_app/3]).
+         create_config/2, create_config/3, package_app/4]).
 -export([create_random_name/1, create_random_vsn/0, write_src_file/2,
          random_element/1]).
 
@@ -470,27 +470,24 @@ get_app_metadata(Name, Vsn, Deps) ->
       {registered, []},
       {applications, Deps}]}.
 
-package_app(AppDir, DestDir, PkgName) ->
-    Name = PkgName++".tar",
-    {ok, Fs} = rebar_utils:list_dir(AppDir),
-    ok = erl_tar:create(filename:join(DestDir, "contents.tar.gz"),
-                        lists:zip(Fs, [filename:join(AppDir,F) || F <- Fs]),
-                        [compressed]),
-    ok = file:write_file(filename:join(DestDir, "metadata.config"), "who cares"),
-    ok = file:write_file(filename:join(DestDir, "VERSION"), "3"),
-    {ok, Contents} = file:read_file(filename:join(DestDir, "contents.tar.gz")),
-    Blob = <<"3who cares", Contents/binary>>,
-    <<X:256/big-unsigned>> = crypto:hash(sha256, Blob),
-    BinChecksum = list_to_binary(rebar_string:uppercase(lists:flatten(io_lib:format("~64.16.0b", [X])))),
-    ok = file:write_file(filename:join(DestDir, "CHECKSUM"), BinChecksum),
-    PkgFiles = ["contents.tar.gz", "VERSION", "metadata.config", "CHECKSUM"],
+package_app(AppDir, DestDir, PkgName, PkgVsn) ->
+    AppSrc = filename:join(AppDir, "src"),
+    {ok, Fs} = rebar_utils:list_dir(AppSrc),
+    Files = lists:zip([filename:join("src", F) || F <- Fs], [filename:join(AppSrc,F) || F <- Fs]),
+    Metadata = #{<<"app">> => list_to_binary(PkgName),
+                 <<"version">> => list_to_binary(PkgVsn)},
+    {ok, {Tarball, <<Checksum:256/big-unsigned-integer>>}} = hex_tarball:create(Metadata, Files),
+
+    Name = PkgName++"-"++PkgVsn++".tar",
     Archive = filename:join(DestDir, Name),
-    ok = erl_tar:create(Archive,
-                        lists:zip(PkgFiles, [filename:join(DestDir,F) || F <- PkgFiles])),
-    {ok, BinFull} = file:read_file(Archive),
-    <<E:128/big-unsigned-integer>> = crypto:hash(md5, BinFull),
-    Etag = rebar_string:lowercase(lists:flatten(io_lib:format("~32.16.0b", [E]))),
-    {BinChecksum, Etag}.
+    file:write_file(Archive, Tarball),
+
+    <<E:128/big-unsigned-integer>> = crypto:hash(md5, Tarball),
+
+    Checksum1 = list_to_binary(
+                  rebar_string:uppercase(
+                    lists:flatten(io_lib:format("~64.16.0b", [Checksum])))),
+    {Checksum1, E}.
 
 random_element(Repos) ->
     Index = ?random:uniform(length(Repos)),