From c0eab91855029f9a868580430d9b902d40a97e60 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Thu, 13 Sep 2018 12:33:17 -0600 Subject: [PATCH] use hex_core tarball unpacking support in pkg resource (#1883) * use hex_core tarball unpacking support in pkg resource * ignore etag if package doesn't exist and delete if checksum fails * add back tests for bad package checksums * improve bad registry checksum error message --- src/rebar_fetch.erl | 16 -- src/rebar_pkg_resource.erl | 168 ++++++++---------- test/mock_pkg_resource.erl | 21 +-- test/rebar_pkg_SUITE.erl | 55 ++++-- .../badindexchk-1.0.0.tar | Bin 10240 -> 10240 bytes test/rebar_pkg_SUITE_data/badpkg-1.0.0.tar | Bin 10240 -> 10240 bytes test/rebar_pkg_SUITE_data/goodpkg-1.0.0.tar | Bin 10240 -> 10240 bytes test/rebar_pkg_alias_SUITE.erl | 11 +- test/rebar_test_utils.erl | 39 ++-- 9 files changed, 150 insertions(+), 160 deletions(-) diff --git a/src/rebar_fetch.erl b/src/rebar_fetch.erl index 4a60864f..9c76e0ea 100644 --- a/src/rebar_fetch.erl +++ b/src/rebar_fetch.erl @@ -71,26 +71,10 @@ needs_update(AppInfo, State) -> true end. -format_error({bad_download, CachePath}) -> - io_lib:format("Download of package does not match md5sum from server: ~ts", [CachePath]); -format_error({unexpected_hash, CachePath, Expected, Found}) -> - io_lib:format("The checksum for package at ~ts (~ts) does not match the " - "checksum previously locked (~ts). Either unlock or " - "upgrade the package, or make sure you fetched it from " - "the same index from which it was initially fetched.", - [CachePath, Found, Expected]); -format_error({failed_extract, CachePath}) -> - io_lib:format("Failed to extract package: ~ts", [CachePath]); -format_error({bad_etag, Source}) -> - io_lib:format("MD5 Checksum comparison failed for: ~ts", [Source]); format_error({fetch_fail, Name, Vsn}) -> io_lib:format("Failed to fetch and copy dep: ~ts-~ts", [Name, Vsn]); format_error({fetch_fail, Source}) -> io_lib:format("Failed to fetch and copy dep: ~p", [Source]); -format_error({bad_checksum, File}) -> - io_lib:format("Checksum mismatch against tarball in ~ts", [File]); -format_error({bad_registry_checksum, File}) -> - io_lib:format("Checksum mismatch against registry in ~ts", [File]); format_error({dep_app_not_found, AppName}) -> io_lib:format("Dependency failure: source for ~ts does not contain a " "recognizable project and can not be built", [AppName]). diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl index d31c4af0..97eabb6e 100644 --- a/src/rebar_pkg_resource.erl +++ b/src/rebar_pkg_resource.erl @@ -9,20 +9,18 @@ download/4, download/5, needs_update/2, - make_vsn/2]). + make_vsn/2, + format_error/1]). -ifdef(TEST). %% exported for test purposes --export([store_etag_in_cache/2, etag/1, request/4]). +-export([store_etag_in_cache/2]). -endif. -include("rebar.hrl"). +-include_lib("providers/include/providers.hrl"). --type cached_result() :: ok | - {bad_checksum,string()} | - {bad_registry_checksum,string()} | - {failed_extract,string()} | - {unexpected_hash,string(),_,binary()}. +-type package() :: {pkg, binary(), binary(), binary(), rebar_hex_repos:repo()}. %%============================================================================== %% Public API @@ -97,19 +95,27 @@ download(TmpDir, AppInfo, State, ResourceState) -> %%------------------------------------------------------------------------------ -spec download(TmpDir, Pkg, State, ResourceState, UpdateETag) -> Res when TmpDir :: file:name(), - Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary(), RepoConfig :: rebar_hex_repos:repo()}, + Pkg :: package(), State :: rebar_state:t(), ResourceState:: rebar_resource_v2:resource_state(), UpdateETag :: boolean(), - Res :: ok | {error,_} | cached_result() | {fetch_fail, binary(), binary()} | {bad_download, file:name()}. + Res :: ok | {error,_} | {unexpected_hash, string(), integer(), integer()} | + {fetch_fail, binary(), binary()}. download(TmpDir, Pkg={pkg, Name, Vsn, _Hash, Repo}, State, _ResourceState, UpdateETag) -> {ok, PackageDir} = rebar_packages:package_dir(Repo, State), Package = binary_to_list(<>), ETagFile = binary_to_list(<>), CachePath = filename:join(PackageDir, Package), ETagPath = filename:join(PackageDir, ETagFile), - cached_download(TmpDir, CachePath, Pkg, etag(ETagPath), - State, ETagPath, UpdateETag). + case cached_download(TmpDir, CachePath, Pkg, etag(CachePath, ETagPath), ETagPath, UpdateETag) of + {bad_registry_checksum, Expected, Found} -> + %% checksum comparison failed. in case this is from a modified cached package + %% overwrite the etag if it exists so it is not relied on again + store_etag_in_cache(ETagPath, <<>>), + ?PRV_ERROR({bad_registry_checksum, Name, Vsn, Expected, Found}); + Result -> + Result + end. %%------------------------------------------------------------------------------ %% @doc @@ -124,6 +130,12 @@ download(TmpDir, Pkg={pkg, Name, Vsn, _Hash, Repo}, State, _ResourceState, Updat make_vsn(_, _) -> {error, "Replacing version of type pkg not supported."}. +format_error({bad_registry_checksum, Name, Vsn, Expected, Found}) -> + io_lib:format("The checksum for package at ~ts-~ts (~ts) does not match the " + "checksum expected from the registry (~ts). " + "Run `rebar3 do unlock ~ts, update` and then try again.", + [Name, Vsn, Found, Expected, Name]). + %%------------------------------------------------------------------------------ %% @doc %% Download the pkg belonging to the given address. If the etag of the pkg @@ -160,13 +172,21 @@ request(Config, Name, Version, ETag) -> %% returned from the hexpm server. The name is package-vsn.etag. %% @end %%------------------------------------------------------------------------------ --spec etag(Path) -> Res when - Path :: file:name(), +-spec etag(PackagePath, ETagPath) -> Res when + PackagePath :: file:name(), + ETagPath :: file:name(), Res :: binary(). -etag(Path) -> - case file:read_file(Path) of +etag(PackagePath, ETagPath) -> + case file:read_file(ETagPath) of {ok, Bin} -> - Bin; + %% just in case a user deleted a cached package but not its etag + %% verify the package is also there, and if not, ignore the etag + case filelib:is_file(PackagePath) of + true -> + Bin; + false -> + <<>> + end; {error, _} -> <<>> end. @@ -186,115 +206,69 @@ store_etag_in_cache(Path, ETag) -> %%%============================================================================= %%% Private functions %%%============================================================================= --spec cached_download(TmpDir, CachePath, Pkg, ETag, State, ETagPath, - UpdateETag) -> Res when +-spec cached_download(TmpDir, CachePath, Pkg, ETag, ETagPath, UpdateETag) -> Res when TmpDir :: file:name(), CachePath :: file:name(), - Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary(), RepoConfig :: rebar_hex_repos:repo()}, + Pkg :: package(), ETag :: binary(), - State :: rebar_state:t(), ETagPath :: file:name(), UpdateETag :: boolean(), - Res :: cached_result() | {fetch_fail, binary(), binary()} | {bad_download, file:name()}. + Res :: ok | {unexpected_hash, integer(), integer()} | {fetch_fail, binary(), binary()}. cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn, _Hash, RepoConfig}, ETag, - State, ETagPath, UpdateETag) -> + ETagPath, UpdateETag) -> case request(RepoConfig, Name, Vsn, ETag) of {ok, cached} -> ?INFO("Version cached at ~ts is up to date, reusing it", [CachePath]), - serve_from_cache(TmpDir, CachePath, Pkg, State); + serve_from_cache(TmpDir, CachePath, Pkg); {ok, Body, NewETag} -> ?INFO("Downloaded package, caching at ~ts", [CachePath]), maybe_store_etag_in_cache(UpdateETag, ETagPath, NewETag), - serve_from_download(TmpDir, CachePath, Pkg, NewETag, Body, - State, ETagPath); + serve_from_download(TmpDir, CachePath, Pkg, Body); error when ETag =/= <<>> -> store_etag_in_cache(ETagPath, ETag), ?INFO("Download error, using cached file at ~ts", [CachePath]), - serve_from_cache(TmpDir, CachePath, Pkg, State); + serve_from_cache(TmpDir, CachePath, Pkg); error -> {fetch_fail, Name, Vsn} end. --spec serve_from_cache(TmpDir, CachePath, Pkg, State) -> Res when +-spec serve_from_cache(TmpDir, CachePath, Pkg) -> Res when TmpDir :: file:name(), CachePath :: file:name(), - Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary(), RepoConfig :: rebar_hex_repos:repo()}, - State :: rebar_state:t(), - Res :: cached_result(). -serve_from_cache(TmpDir, CachePath, Pkg, State) -> - {Files, Contents, Version, Meta} = extract(TmpDir, CachePath), - case checksums(Pkg, Files, Contents, Version, Meta, State) of - {Chk, Chk, Chk} -> - erl_tar:extract({binary, Contents}, [{cwd, TmpDir}, compressed]); - {_Hash, Chk, Chk} -> - ?DEBUG("Expected hash ~p does not match checksums ~p", [_Hash, Chk]), - {unexpected_hash, CachePath, _Hash, Chk}; - {Chk, _Bin, Chk} -> - ?DEBUG("Checksums: registry: ~p, pkg: ~p", [Chk, _Bin]), - {failed_extract, CachePath}; - {_Hash, _Bin, _Tar} -> - ?DEBUG("Checksums: expected: ~p, pkg: ~p, meta: ~p", - [_Hash, _Bin, _Tar]), - {bad_checksum, CachePath} + Pkg :: package(), + Res :: ok | {error,_} | {bad_registry_checksum, integer(), integer()}. +serve_from_cache(TmpDir, CachePath, Pkg) -> + {ok, Binary} = file:read_file(CachePath), + serve_from_memory(TmpDir, Binary, Pkg). + +-spec serve_from_memory(TmpDir, Tarball, Package) -> Res when + TmpDir :: file:name(), + Tarball :: binary(), + Package :: package(), + Res :: ok | {error,_} | {bad_registry_checksum, integer(), integer()}. +serve_from_memory(TmpDir, Binary, {pkg, _Name, _Vsn, Hash, _RepoConfig}) -> + RegistryChecksum = list_to_integer(binary_to_list(Hash), 16), + case hex_tarball:unpack(Binary, TmpDir) of + {ok, #{checksum := <>}} when RegistryChecksum =/= Checksum -> + ?DEBUG("Expected hash ~64.16.0B does not match checksum of fetched package ~64.16.0B", + [RegistryChecksum, Checksum]), + {bad_registry_checksum, RegistryChecksum, Checksum}; + {ok, #{checksum := <>}} -> + ok; + {error, Reason} -> + {error, {hex_tarball, Reason}} end. --spec serve_from_download(TmpDir, CachePath, Package, ETag, Binary, State, - ETagPath) -> Res when +-spec serve_from_download(TmpDir, CachePath, Package, Binary) -> Res when TmpDir :: file:name(), CachePath :: file:name(), - Package :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary(), RepoConfig :: rebar_hex_repos:repo()}, - ETag :: binary(), + Package :: package(), Binary :: binary(), - State :: rebar_state:t(), - ETagPath :: file:name(), - Res :: ok | {error,_} | {bad_download, file:name()}. -serve_from_download(TmpDir, CachePath, Package, ETag, Binary, State, ETagPath) -> + Res :: ok | {error,_}. +serve_from_download(TmpDir, CachePath, Package, Binary) -> ?DEBUG("Writing ~p to cache at ~ts", [Package, CachePath]), file:write_file(CachePath, Binary), - case etag(ETagPath) of - ETag -> - serve_from_cache(TmpDir, CachePath, Package, State); - FileETag -> - ?DEBUG("Downloaded file ~ts ETag ~ts doesn't match returned ETag ~ts", - [CachePath, ETag, FileETag]), - {bad_download, CachePath} - end. - --spec extract(TmpDir, CachePath) -> Res when - TmpDir :: file:name(), - CachePath :: file:name(), - Res :: {Files, Contents, Version, Meta}, - Files :: list({file:name(), binary()}), - Contents :: binary(), - Version :: binary(), - Meta :: binary(). -extract(TmpDir, CachePath) -> - ec_file:mkdir_p(TmpDir), - {ok, Files} = erl_tar:extract(CachePath, [memory]), - {"contents.tar.gz", Contents} = lists:keyfind("contents.tar.gz", 1, Files), - {"VERSION", Version} = lists:keyfind("VERSION", 1, Files), - {"metadata.config", Meta} = lists:keyfind("metadata.config", 1, Files), - {Files, Contents, Version, Meta}. - --spec checksums(Pkg, Files, Contents, Version, Meta, State) -> Res when - Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary(), RepoConfig :: rebar_hex_repos:repo()}, - Files :: list({file:name(), binary()}), - Contents :: binary(), - Version :: binary(), - Meta :: binary(), - State :: rebar_state:t(), - Res :: {Hash, BinChecksum, TarChecksum}, - Hash :: binary(), - BinChecksum :: binary(), - TarChecksum :: binary(). -checksums({pkg, _Name, _Vsn, Hash, _}, Files, Contents, Version, Meta, _State) -> - Blob = <>, - <> = crypto:hash(sha256, Blob), - BinChecksum = list_to_binary( - rebar_string:uppercase( - lists:flatten(io_lib:format("~64.16.0b", [X])))), - {"CHECKSUM", TarChecksum} = lists:keyfind("CHECKSUM", 1, Files), - {Hash, BinChecksum, TarChecksum}. + serve_from_memory(TmpDir, Binary, Package). -spec maybe_store_etag_in_cache(UpdateETag, Path, ETag) -> Res when UpdateETag :: boolean(), diff --git a/test/mock_pkg_resource.erl b/test/mock_pkg_resource.erl index 4b23497c..9a314610 100644 --- a/test/mock_pkg_resource.erl +++ b/test/mock_pkg_resource.erl @@ -93,20 +93,21 @@ mock_download(Opts) -> [kernel, stdlib] ++ [element(1,D) || D <- AppDeps] ), rebar_test_utils:create_config(Dir, [{deps, AppDeps}]++Config), + TarApp = App++"-"++rebar_utils:to_list(Vsn)++".tar", - Tarball = filename:join([Dir, TarApp]), - Contents = filename:join([Dir, "contents.tar.gz"]), + + Metadata = #{<<"app">> => AppBin, + <<"version">> => Vsn}, + Files = all_files(rebar_app_info:dir(AppInfo1)), - ok = erl_tar:create(Contents, - archive_names(Dir, App, Vsn, Files), - [compressed]), - ok = erl_tar:create(Tarball, - [{"contents.tar.gz", Contents}], - []), + {ok, {Tarball, _Checksum}} = hex_tarball:create(Metadata, archive_names(Dir, Files)), + Archive = filename:join([Dir, TarApp]), + file:write_file(Archive, Tarball), + Cache = proplists:get_value(cache_dir, Opts, filename:join(Dir,"cache")), Cached = filename:join([Cache, TarApp]), filelib:ensure_dir(Cached), - rebar_file_utils:mv(Tarball, Cached), + rebar_file_utils:mv(Archive, Cached), ok end). @@ -137,7 +138,7 @@ mock_pkg_index(Opts) -> all_files(Dir) -> filelib:wildcard(filename:join([Dir, "**"])). -archive_names(Dir, _App, _Vsn, Files) -> +archive_names(Dir, Files) -> [{(F -- Dir) -- "/", F} || F <- Files]. find_parts(Apps, Skip) -> find_parts(Apps, Skip, dict:new()). diff --git a/test/rebar_pkg_SUITE.erl b/test/rebar_pkg_SUITE.erl index 1085f5fc..62bc4d11 100644 --- a/test/rebar_pkg_SUITE.erl +++ b/test/rebar_pkg_SUITE.erl @@ -8,14 +8,15 @@ -define(bad_etag, <<"abcdef">>). -define(good_etag, <<"22e1d7387c9085a462340088a2a8ba67">>). +-define(badpkg_checksum, <<"A14E3718B33F8124E98004433193509EC6660F6CA03302657CAB8785751D77A0">>). +-define(badindex_checksum, <<"7B2CBED315C89F3126B5BF553DD7FF0FB5FE94B064888DD1B095CE8BF4B6A16A">>). -define(bad_checksum, <<"D576B442A68C7B92BACDE1EFE9C6E54D8D6C74BDB71D8175B9D3C6EC8C7B62A7">>). --define(good_checksum, <<"1C6CE379D191FBAB41B7905075E0BF87CBBE23C77CECE775C5A0B786B2244C35">>). +-define(good_checksum, <<"12726BDE1F65583A0817A7E8AADCA73F03FD8CB06F01E6CD29117C4A0DA0AFCF">>). -define(BADPKG_ETAG, <<"BADETAG">>). -all() -> [good_uncached, good_cached, badpkg, - %% badindexchk, badhash_nocache, badhash_cache, - bad_to_good, good_disconnect, bad_disconnect, pkgs_provider, - find_highest_matching]. +all() -> [good_uncached, good_cached, badpkg, badhash_nocache, + badindexchk, badhash_cache, bad_to_good, good_disconnect, + bad_disconnect, pkgs_provider, find_highest_matching]. init_per_suite(Config) -> application:start(meck), @@ -96,9 +97,6 @@ init_per_testcase(bad_disconnect=Name, Config0) -> {pkg, Pkg} | Config0], Config = mock_config(Name, Config1), - %% meck:unload(httpc), - %% meck:new(httpc, [passthrough, unsticky]), - %% meck:expect(httpc, request, fun(_, _, _, _) -> {error, econnrefused} end), meck:expect(hex_repo, get_tarball, fun(_, _, _) -> {error, econnrefused} end), @@ -134,6 +132,17 @@ good_cached(Config) -> rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum, #{}}, State, #{}, true)), {ok, Content} = file:read_file(CachedFile). + +badindexchk(Config) -> + Tmp = ?config(tmp_dir, Config), + {Pkg,Vsn} = ?config(pkg, Config), + State = ?config(state, Config), + ?assertMatch({error, {rebar_pkg_resource, {bad_registry_checksum, _, _, _, _}}}, + rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?bad_checksum, #{}}, State, #{}, true)), + %% The cached file is there for forensic purposes + Cache = ?config(cache_dir, Config), + ?assert(filelib:is_regular(filename:join(Cache, <>))). + badpkg(Config) -> Tmp = ?config(tmp_dir, Config), {Pkg,Vsn} = ?config(pkg, Config), @@ -142,12 +151,36 @@ badpkg(Config) -> CachePath = filename:join(Cache, <>), ETagPath = filename:join(Cache, <>), rebar_pkg_resource:store_etag_in_cache(ETagPath, ?BADPKG_ETAG), - ?assertMatch({bad_download, _Path}, - rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum, #{}}, State, #{}, false)), + ?assertMatch({error, {hex_tarball, {tarball, {checksum_mismatch, _, _}}}}, + rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?badpkg_checksum, #{}}, State, #{}, false)), %% The cached/etag files are there for forensic purposes ?assert(filelib:is_regular(ETagPath)), ?assert(filelib:is_regular(CachePath)). +badhash_nocache(Config) -> + Tmp = ?config(tmp_dir, Config), + {Pkg,Vsn} = ?config(pkg, Config), + State = ?config(state, Config), + ?assertMatch({error, {rebar_pkg_resource, {bad_registry_checksum, _, _, _, _}}}, + rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?bad_checksum, #{}}, State, #{}, true)), + %% The cached file is there for forensic purposes + Cache = ?config(cache_dir, Config), + ?assert(filelib:is_regular(filename:join(Cache, <>))). + +badhash_cache(Config) -> + Tmp = ?config(tmp_dir, Config), + {Pkg,Vsn} = ?config(pkg, Config), + Cache = ?config(cache_dir, Config), + State = ?config(state, Config), + CachedFile = filename:join(Cache, <>), + ?assert(filelib:is_regular(CachedFile)), + {ok, Content} = file:read_file(CachedFile), + ?assertMatch({error, {rebar_pkg_resource, {bad_registry_checksum, _, _, _, _}}}, + rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?bad_checksum, #{}}, State, #{}, true)), + %% The cached file is there still, unchanged. + ?assert(filelib:is_regular(CachedFile)), + ?assertEqual({ok, Content}, file:read_file(CachedFile)). + bad_to_good(Config) -> Tmp = ?config(tmp_dir, Config), {Pkg,Vsn} = ?config(pkg, Config), @@ -220,7 +253,7 @@ mock_config(Name, Config) -> {{<<"goodpkg">>,<<"1.0.1">>}, [[], ?good_checksum, [<<"rebar3">>]]}, {{<<"goodpkg">>,<<"1.1.1">>}, [[], ?good_checksum, [<<"rebar3">>]]}, {{<<"goodpkg">>,<<"2.0.0">>}, [[], ?good_checksum, [<<"rebar3">>]]}, - {{<<"badpkg">>,<<"1.0.0">>}, [[], ?good_checksum, [<<"rebar3">>]]} + {{<<"badpkg">>,<<"1.0.0">>}, [[], ?badpkg_checksum, [<<"rebar3">>]]} ], ets:insert_new(Tid, AllDeps), CacheDir = filename:join([CacheRoot, "hex", "com", "test", "packages"]), diff --git a/test/rebar_pkg_SUITE_data/badindexchk-1.0.0.tar b/test/rebar_pkg_SUITE_data/badindexchk-1.0.0.tar index e5b963f4fead7c9b249dca3cf94644562a01b881..1765bb3c773c71a285e5fd7ca64747bd04ebe1ad 100644 GIT binary patch delta 427 zcmZn&Xb6~4q;0@pU|?WqZVIM>Od!X=kipQ{*u>1#z|6?lh{3?n2*_nnFqmw}n8IIL zT#{I%pukX4lnG?zO%#-5hcYKSGF@;qcQSHza&<8_G zBLm{>-^5nLhQ&(;=4LQ2fjkKGlCim|32{c+RomDoB^DGY+1cqR0GUaNDVcdGsTIi? z*+8LMJuZ-ZS!z*nW_}(_jiH`_o&iL5vm)Oi#z_K2$X*hVW;Zf0H6_meMM7c#ZT4`K delta 371 zcmaivF;2rk5JlI>N(7<^DQJ>nI|`ba-JPAd!@F!Gx?I7+Mxg;|ZA3wnBS3rujsO?n z4!J@O!U7T!5|XR`=NEte+pIRL=Xj0+0FdchlT8PJKt;+hve8m3vB0G(ok7P15HJOB%LRMqg3_MZ2OpTpQwfe2)+uF- z*YuuEKeBBe4}TZ+Px)-D2-VQ1s05-6(NNj^O4j#xXPvyRVj~~d fRVTakZJWZTJxU|@*>>`0dfJDk@U~MP>l&B~WP+>T}e4EV+ zCqyuJ6R)PL&UNcwR>|4VeTph4Mw=21ElQQg7C6#}-0Ut+!lDkpFn(Mu^i+;aXlx1%)xD#GJ z7US~e*}ISEVi&zXx!L4(dWbQC2==KT^Bu0+ryzUG2gr!Nn}-Zq*6S0}rqlhsT8Z`_ De%5ql delta 393 zcmah_F;2rk5VWl*2t)x$RHV3$f@b$__s(e|oi|3J%NHzc6dI7uMiex80^pyJr|<>d zfj9?270K1iPBF7Ho7HCZ8m=$_0Cc>=NaF#Rpk_@DEZL|dSObZI9@F~xsqLzuRC3pZ z`t5W!8L;=+=K{zhQ(nkY2%Qxv8H!vi0s8WKX^D}%ltqqnM(?$tA;}%c+!4ukkx_Q| z|4jcfxj4{_dMx&18m445|Z>4m7NH9fvxt|m4N zy_KCiT0Ebe2R~S||G^}*NJe=&E0i~LCzn-3b0zj&6+yDRo delta 371 zcmaivF;2rk5JlI>N(7<^DQJ>nI|`ba-JPAd!@F!Gx?I7+Mxg;|ZA3wnBS3rujsO?n z4!J@O!U7T!5|XR`=NEte+pIRL=Xj0+0FdchlT8PJKt;+hve8m3vB0G(ok7P15HJOB%LRMqg3_MZ2OpTpQwfe2)+uF- z*YuuEKeBBe4}TZ+Px)-D2-VQ1s05-6(NNj^O4j#xXPvyRVj~~d fRVTakZJWZTJxU|@*>>`0 CacheDir = filename:join([CacheRoot, "hex", "com", "test", "packages"]), rebar_test_utils:create_app(AppDir, "fakelib", "1.0.0", [kernel, stdlib]), ct:pal("{~p, ~p}",[ChkFake, Etag]), - {ChkFake, Etag} = rebar_test_utils:package_app(AppDir, CacheDir, "goodpkg-1.0.0"), + {ChkGood, EtagGood} = rebar_test_utils:package_app(AppDir, CacheDir, "goodpkg", "1.0.0"), AllDeps = [ {<<"fakelib">>,[[<<"1.0.0">>]]}, @@ -198,7 +198,7 @@ mock_config(Name, Config) -> {<<"topdep">>,[[<<"1.0.0">>]]}, {<<"transitive">>, [[<<"1.0.0">>]]}, {{<<"fakelib">>,<<"1.0.0">>}, [[], ChkFake, [<<"rebar3">>]]}, - {{<<"goodpkg">>,<<"1.0.0">>}, [[], ChkFake, [<<"rebar3">>]]}, + {{<<"goodpkg">>,<<"1.0.0">>}, [[], ChkGood, [<<"rebar3">>]]}, {{<<"topdep">>,<<"1.0.0">>}, [[ {<<"transitive">>, <<"1.0.0">>, false, <<"transitive_app">>} @@ -248,13 +248,14 @@ mock_config(Name, Config) -> Releases = [#{checksum => Checksum, version => Vsn, - dependencies => [{DAppName, {pkg, DN, DV, undefined}} || {DN, DV, _, DAppName} <- Deps]} || + dependencies => [{DAppName, {pkg, DN, DV, undefined}} || + {DN, DV, _, DAppName} <- Deps]} || {{_, Vsn}, [Deps, Checksum, _]} <- Matches], {ok, {200, #{}, #{releases => Releases}}} end), meck:expect(hex_repo, get_tarball, fun(_, _, _) -> - {ok, {304, #{<<"etag">> => Etag}, <<>>}} + {ok, {304, #{<<"etag">> => EtagGood}, <<>>}} end), %% Move all packages to cache @@ -278,4 +279,4 @@ create_lib(Name, Config, AppName, PkgName) -> CacheDir = filename:join([CacheRoot, "hex", "com", "test", "packages"]), filelib:ensure_dir(filename:join([CacheDir, "registry"])), rebar_test_utils:create_app(AppDir, AppName, "1.0.0", [kernel, stdlib]), - rebar_test_utils:package_app(AppDir, CacheDir, PkgName++"-1.0.0"). + rebar_test_utils:package_app(AppDir, CacheDir, PkgName, "1.0.0"). diff --git a/test/rebar_test_utils.erl b/test/rebar_test_utils.erl index 7de4899b..8bcb6d12 100644 --- a/test/rebar_test_utils.erl +++ b/test/rebar_test_utils.erl @@ -4,7 +4,7 @@ -export([init_rebar_state/1, init_rebar_state/2, run_and_check/4, check_results/3]). -export([expand_deps/2, flat_deps/1, top_level_deps/1]). -export([create_app/4, create_plugin/4, create_eunit_app/4, create_empty_app/4, - create_config/2, create_config/3, package_app/3]). + create_config/2, create_config/3, package_app/4]). -export([create_random_name/1, create_random_vsn/0, write_src_file/2, random_element/1]). @@ -470,27 +470,24 @@ get_app_metadata(Name, Vsn, Deps) -> {registered, []}, {applications, Deps}]}. -package_app(AppDir, DestDir, PkgName) -> - Name = PkgName++".tar", - {ok, Fs} = rebar_utils:list_dir(AppDir), - ok = erl_tar:create(filename:join(DestDir, "contents.tar.gz"), - lists:zip(Fs, [filename:join(AppDir,F) || F <- Fs]), - [compressed]), - ok = file:write_file(filename:join(DestDir, "metadata.config"), "who cares"), - ok = file:write_file(filename:join(DestDir, "VERSION"), "3"), - {ok, Contents} = file:read_file(filename:join(DestDir, "contents.tar.gz")), - Blob = <<"3who cares", Contents/binary>>, - <> = crypto:hash(sha256, Blob), - BinChecksum = list_to_binary(rebar_string:uppercase(lists:flatten(io_lib:format("~64.16.0b", [X])))), - ok = file:write_file(filename:join(DestDir, "CHECKSUM"), BinChecksum), - PkgFiles = ["contents.tar.gz", "VERSION", "metadata.config", "CHECKSUM"], +package_app(AppDir, DestDir, PkgName, PkgVsn) -> + AppSrc = filename:join(AppDir, "src"), + {ok, Fs} = rebar_utils:list_dir(AppSrc), + Files = lists:zip([filename:join("src", F) || F <- Fs], [filename:join(AppSrc,F) || F <- Fs]), + Metadata = #{<<"app">> => list_to_binary(PkgName), + <<"version">> => list_to_binary(PkgVsn)}, + {ok, {Tarball, <>}} = hex_tarball:create(Metadata, Files), + + Name = PkgName++"-"++PkgVsn++".tar", Archive = filename:join(DestDir, Name), - ok = erl_tar:create(Archive, - lists:zip(PkgFiles, [filename:join(DestDir,F) || F <- PkgFiles])), - {ok, BinFull} = file:read_file(Archive), - <> = crypto:hash(md5, BinFull), - Etag = rebar_string:lowercase(lists:flatten(io_lib:format("~32.16.0b", [E]))), - {BinChecksum, Etag}. + file:write_file(Archive, Tarball), + + <> = crypto:hash(md5, Tarball), + + Checksum1 = list_to_binary( + rebar_string:uppercase( + lists:flatten(io_lib:format("~64.16.0b", [Checksum])))), + {Checksum1, E}. random_element(Repos) -> Index = ?random:uniform(length(Repos)),