Merge pull request #414 from tsloughter/hex_pkg_improvements

Hex pkg improvements

rebar.config

@@ -30,9 +30,7 @@
                       {"rebar/include/*", "."}]}.
 
 {erl_opts,
- [{platform_define, "R14", no_callback_support},
-  {platform_define, "^[0-9]+", namespaced_types},
-  {platform_define, "^R1[4|5]", deprecated_crypto},
+ [{platform_define, "^[0-9]+", namespaced_types},
   no_debug_info,
   warnings_as_errors]}.
 

src/rebar.hrl

@@ -22,6 +22,7 @@
 -define(DEFAULT_TEST_DEPS_DIR, "test/lib").
 -define(DEFAULT_RELEASE_DIR, "rel").
 -define(DEFAULT_CONFIG_FILE, "rebar.config").
+-define(DEFAULT_CDN, "https://s3.amazonaws.com/s3.hex.pm/tarballs").
 -define(LOCK_FILE, "rebar.lock").
 
 -ifdef(namespaced_types).

src/rebar_fetch.erl

@@ -26,42 +26,34 @@ lock_source(AppDir, Source, State) ->
 -spec download_source(file:filename_all(), rebar_resource:resource(), rebar_state:t()) ->
                              true | {error, any()}.
 download_source(AppDir, Source, State) ->
-    try
-        Resources = rebar_state:resources(State),
-        Module = get_resource_type(Source, Resources),
-        TmpDir = ec_file:insecure_mkdtemp(),
-        AppDir1 = ec_cnv:to_list(AppDir),
-        case Module:download(TmpDir, Source, State) of
-            {ok, _} ->
-                ec_file:mkdir_p(AppDir1),
-                code:del_path(filename:absname(filename:join(AppDir1, "ebin"))),
-                ec_file:remove(filename:absname(AppDir1), [recursive]),
-                ?DEBUG("Moving checkout ~p to ~p", [TmpDir, filename:absname(AppDir1)]),
-                ok = rebar_file_utils:mv(TmpDir, filename:absname(AppDir1)),
-                true;
-            {tarball, File} ->
-                Contents = filename:join(TmpDir, "contents"),
-                ec_file:mkdir_p(AppDir1),
-                ec_file:mkdir_p(Contents),
-                ok = erl_tar:extract(File, [{cwd, TmpDir}]),
-                ok = erl_tar:extract(filename:join(TmpDir, "contents.tar.gz"),
-                                    [{cwd, Contents}, compressed]),
-                code:del_path(filename:absname(filename:join(AppDir1, "ebin"))),
-                ec_file:remove(filename:absname(AppDir1), [recursive]),
-
-                ?DEBUG("Moving contents ~p to ~p", [Contents, filename:absname(AppDir1)]),
-                ok = rebar_file_utils:mv(Contents, filename:absname(AppDir1)),
-
-                ?DEBUG("Removing tmp dir ~p", [TmpDir]),
-                ec_file:remove(TmpDir, [recursive]),
-                true
-        end
+    try download_source_(AppDir, Source, State) of
+        true ->
+            true;
+        Error ->
+            throw(?PRV_ERROR(Error))
     catch
         C:T ->
             ?DEBUG("rebar_fetch exception ~p ~p ~p", [C, T, erlang:get_stacktrace()]),
             throw(?PRV_ERROR({fetch_fail, Source}))
     end.
 
+download_source_(AppDir, Source, State) ->
+    Resources = rebar_state:resources(State),
+    Module = get_resource_type(Source, Resources),
+    TmpDir = ec_file:insecure_mkdtemp(),
+    AppDir1 = ec_cnv:to_list(AppDir),
+    case Module:download(TmpDir, Source, State) of
+        {ok, _} ->
+            ec_file:mkdir_p(AppDir1),
+            code:del_path(filename:absname(filename:join(AppDir1, "ebin"))),
+            ec_file:remove(filename:absname(AppDir1), [recursive]),
+            ?DEBUG("Moving checkout ~p to ~p", [TmpDir, filename:absname(AppDir1)]),
+            ok = rebar_file_utils:mv(TmpDir, filename:absname(AppDir1)),
+            true;
+        Error ->
+            Error
+    end.
+
 -spec needs_update(file:filename_all(), rebar_resource:resource(), rebar_state:t()) -> boolean() | {error, string()}.
 needs_update(AppDir, Source, State) ->
     Resources = rebar_state:resources(State),
@@ -73,8 +65,18 @@ needs_update(AppDir, Source, State) ->
             true
     end.
 
+format_error({bad_download, CachePath}) ->
+    io_lib:format("Download of package does not match md5sum from server: ~s", [CachePath]);
+format_error({failed_extract, CachePath}) ->
+    io_lib:format("Failed to extract package: ~s", [CachePath]);
+format_error({bad_etag, Source}) ->
+    io_lib:format("MD5 Checksum comparison failed for: ~s", [Source]);
 format_error({fetch_fail, Source}) ->
-    io_lib:format("Failed to fetch and copy dep: ~p", [Source]).
+    io_lib:format("Failed to fetch and copy dep: ~s", [Source]);
+format_error({bad_checksum, File}) ->
+    io_lib:format("Checksum mismatch against tarball in ~s", [File]);
+format_error({bad_registry_checksum, File}) ->
+    io_lib:format("Checksum mismatch against registry in ~s", [File]).
 
 get_resource_type({Type, Location}, Resources) ->
     find_resource_module(Type, Location, Resources);

src/rebar_packages.erl

@@ -2,7 +2,9 @@
 
 -export([get_packages/1
         ,registry/1
+        ,package_dir/1
         ,check_registry/3
+        ,registry_checksum/2
         ,find_highest_matching/3]).
 
 -export_type([package/0]).
@@ -15,8 +17,7 @@
 
 -spec get_packages(rebar_state:t()) -> {rebar_dict(), rebar_digraph()}.
 get_packages(State) ->
-    RebarDir = rebar_dir:global_cache_dir(State),
-    RegistryDir = filename:join(RebarDir, "packages"),
+    RegistryDir = package_dir(State),
     DictFile = filename:join(RegistryDir, "dict"),
     Edges = filename:join(RegistryDir, "edges"),
     Vertices = filename:join(RegistryDir, "vertices"),
@@ -42,8 +43,7 @@ get_packages(State) ->
     end.
 
 registry(State) ->
-    Dir = rebar_dir:global_cache_dir(State),
-    RegistryDir = filename:join(Dir, "packages"),
+    RegistryDir = package_dir(State),
     HexFile = filename:join(RegistryDir, "registry"),
     case ets:file2tab(HexFile) of
         {ok, T} ->
@@ -53,6 +53,17 @@ registry(State) ->
             error
     end.
 
+package_dir(State) ->
+    CacheDir = rebar_dir:global_cache_dir(State),
+    CDN = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN),
+    {ok, {_, _, Host, _, Path, _}} = http_uri:parse(CDN),
+    CDNHostPath = lists:reverse(string:tokens(Host, ".")),
+    CDNPath = tl(filename:split(Path)),
+    PackageDir = filename:join([CacheDir, "hex"] ++ CDNHostPath ++ CDNPath ++ ["packages"]),
+    ok = filelib:ensure_dir(filename:join(PackageDir, "placeholder")),
+    PackageDir.
+
+
 check_registry(Pkg, Vsn, State) ->
     case rebar_state:registry(State) of
         {ok, T} ->
@@ -66,6 +77,15 @@ check_registry(Pkg, Vsn, State) ->
             false
     end.
 
+registry_checksum({pkg, Name, Vsn}, State) ->
+    {ok, Registry} = registry(State),
+    case ets:lookup(Registry, {Name, Vsn}) of
+        [{{_, _}, [_, Checksum | _]}] ->
+            Checksum;
+        [] ->
+            none
+    end.
+
 %% Hex supports use of ~> to specify the version required for a dependency.
 %% Since rebar3 requires exact versions to choose from we find the highest
 %% available version of the dep that passes the constraint.

src/rebar_pkg_resource.erl

@@ -23,12 +23,99 @@ needs_update(Dir, {pkg, _Name, Vsn}) ->
             true
     end.
 
-download(Dir, {pkg, Name, Vsn}, State) ->
-    TmpFile = filename:join(Dir, "package.tar.gz"),
-    CDN = rebar_state:get(State, rebar_packages_cdn, "https://s3.amazonaws.com/s3.hex.pm/tarballs"),
-    Url = string:join([CDN, binary_to_list(<<Name/binary, "-", Vsn/binary, ".tar">>)], "/"),
-    {ok, saved_to_file} = httpc:request(get, {Url, []}, [], [{stream, TmpFile}]),
-    {tarball, TmpFile}.
+download(TmpDir, Pkg={pkg, Name, Vsn}, State) ->
+    CDN = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN),
+    PackageDir = rebar_packages:package_dir(State),
+    Package = binary_to_list(<<Name/binary, "-", Vsn/binary, ".tar">>),
+    CachePath = filename:join(PackageDir, Package),
+    Url = string:join([CDN, Package], "/"),
+    cached_download(TmpDir, CachePath, Pkg, Url, etag(CachePath), State).
+
+cached_download(TmpDir, CachePath, Pkg, Url, ETag, State) ->
+    case request(Url, ETag) of
+        {ok, cached} ->
+            serve_from_cache(TmpDir, CachePath, Pkg, State);
+        {ok, Body, NewETag} ->
+            serve_from_download(TmpDir, CachePath, Pkg, NewETag, Body, State);
+        error when ETag =/= false ->
+            ?DEBUG("Download ~s error, using ~s from cache", [Url, CachePath]),
+            serve_from_cache(TmpDir, CachePath, Pkg, State);
+        error ->
+            request_failed
+    end.
+
+serve_from_cache(TmpDir, CachePath, Pkg, State) ->
+    {Files, Contents, Version, Meta} = extract(TmpDir, CachePath),
+    case checksums(Pkg, Files, Contents, Version, Meta, State) of
+        {Chk, Chk, Chk} ->
+            ok = erl_tar:extract({binary, Contents}, [{cwd, TmpDir}, compressed]),
+            {ok, true};
+        {_Bin, Chk, Chk} ->
+            ?DEBUG("Checksums: registry: ~p, pkg: ~p", [Chk, _Bin]),
+            {failed_extract, CachePath};
+        {Chk, _Reg, Chk} ->
+            ?DEBUG("Checksums: registry: ~p, pkg: ~p", [_Reg, Chk]),
+            {bad_registry_checksum, CachePath};
+        {_Bin, _Reg, _Tar} ->
+            ?DEBUG("Checksums: registry: ~p, pkg: ~p, meta: ~p", [_Reg, _Bin, _Tar]),
+            {bad_checksum, CachePath}
+    end.
+
+serve_from_download(TmpDir, CachePath, Package, ETag, Binary, State) ->
+    ?DEBUG("Writing ~p to cache at ~s", [Package, CachePath]),
+    file:write_file(CachePath, Binary),
+    case etag(CachePath) of
+        ETag ->
+            serve_from_cache(TmpDir, CachePath, Package, State);
+        FileETag ->
+            ?DEBUG("Downloaded file ~s ETag ~s doesn't match returned ETag ~s", [CachePath, ETag, FileETag]),
+            {bad_download, CachePath}
+    end.
+
+
+extract(TmpDir, CachePath) ->
+    ec_file:mkdir_p(TmpDir),
+    {ok, Files} = erl_tar:extract(CachePath, [memory]),
+    {"contents.tar.gz", Contents} = lists:keyfind("contents.tar.gz", 1, Files),
+    {"VERSION", Version} = lists:keyfind("VERSION", 1, Files),
+    {"metadata.config", Meta} = lists:keyfind("metadata.config", 1, Files),
+    {Files, Contents, Version, Meta}.
+
+checksums(Pkg, Files, Contents, Version, Meta, State) ->
+    Blob = <<Version/binary, Meta/binary, Contents/binary>>,
+    <<X:256/big-unsigned>> = crypto:hash(sha256, Blob),
+    BinChecksum = list_to_binary(string:to_upper(lists:flatten(io_lib:format("~64.16.0b", [X])))),
+    RegistryChecksum = rebar_packages:registry_checksum(Pkg, State),
+    {"CHECKSUM", TarChecksum} = lists:keyfind("CHECKSUM", 1, Files),
+    {BinChecksum, RegistryChecksum, TarChecksum}.
 
 make_vsn(_) ->
     {error, "Replacing version of type pkg not supported."}.
+
+request(Url, ETag) ->
+    case httpc:request(get, {Url, [{"if-none-match", ETag} || ETag =/= false]},
+                       [{relaxed, true}],
+                       [{body_format, binary}]) of
+        {ok, {{_Version, 200, _Reason}, Headers, Body}} ->
+            ?DEBUG("Successfully downloaded ~s", [Url]),
+            {"etag", ETag1} = lists:keyfind("etag", 1, Headers),
+            {ok, Body, string:strip(ETag1, both, $")};
+        {ok, {{_Version, 304, _Reason}, _Headers, _Body}} ->
+            ?DEBUG("Cached copy of ~s still valid", [Url]),
+            {ok, cached};
+        {ok, {{_Version, Code, _Reason}, _Headers, _Body}} ->
+            ?DEBUG("Request to ~p failed: status code ~p", [Url, Code]),
+            error;
+        {error, Reason} ->
+            ?DEBUG("Request to ~p failed: ~p", [Url, Reason]),
+            error
+    end.
+
+etag(Path) ->
+    case file:read_file(Path) of
+        {ok, Binary} ->
+            <<X:128/big-unsigned-integer>> = crypto:hash(md5, Binary),
+            string:to_lower(lists:flatten(io_lib:format("~32.16.0b", [X])));
+        {error, _} ->
+            false
+    end.

src/rebar_prv_install_deps.erl

@@ -151,7 +151,7 @@ handle_deps(Profile, State0, Deps, Upgrade, Locks) ->
                                             ,{Packages, Graph}),
 
                 update_pkg_deps(Profile, Packages, PkgDeps1
-                               ,Graph, Upgrade, Seen, State2)
+                               ,Graph, Upgrade, Seen, State2, Locks)
         end,
 
     AllDeps = lists:ukeymerge(2
@@ -184,7 +184,7 @@ find_cycles(Apps) ->
 cull_compile(TopSortedDeps, ProjectApps) ->
     lists:dropwhile(fun not_needs_compile/1, TopSortedDeps -- ProjectApps).
 
-update_pkg_deps(Profile, Packages, PkgDeps, Graph, Upgrade, Seen, State) ->
+update_pkg_deps(Profile, Packages, PkgDeps, Graph, Upgrade, Seen, State, Locks) ->
     case PkgDeps of
         [] -> %% No pkg deps
             {[], State};
@@ -196,13 +196,16 @@ update_pkg_deps(Profile, Packages, PkgDeps, Graph, Upgrade, Seen, State) ->
                 {ok, Solution, []} ->
                     Solution;
                 {ok, Solution, Discarded} ->
-                    [warn_skip_pkg(Pkg, State) || Pkg <- Discarded],
+                    [warn_skip_pkg(Pkg, State) || Pkg <- Discarded, not(pkg_locked(Pkg, Locks))],
                     Solution
             end,
-            update_pkg_deps(Profile, S, Packages, Upgrade, Seen, State)
+            update_pkg_deps(Profile, S, Packages, Upgrade, Seen, State, Locks)
     end.
 
-update_pkg_deps(Profile, Pkgs, Packages, Upgrade, Seen, State) ->
+pkg_locked({Name, _}, Locks) ->
+    false =/= lists:keyfind(Name, 1, Locks).
+
+update_pkg_deps(Profile, Pkgs, Packages, Upgrade, Seen, State, _Locks) ->
     %% Create app_info record for each pkg dep
     DepsDir = profile_dep_dir(State, Profile),
     {Solved, _, State1}
@@ -563,10 +566,10 @@ fetch_app(AppInfo, AppDir, State) ->
     ?INFO("Fetching ~s (~p)", [rebar_app_info:name(AppInfo), rebar_app_info:source(AppInfo)]),
     Source = rebar_app_info:source(AppInfo),
     case rebar_fetch:download_source(AppDir, Source, State) of
-        {error, Reason} ->
-            throw(Reason);
-        Result ->
-            Result
+        true ->
+            true;
+        Error ->
+            throw(Error)
     end.
 
 update_app_info(AppInfo) ->
@@ -587,10 +590,10 @@ maybe_upgrade(AppInfo, AppDir, true, State) ->
         true ->
             ?INFO("Upgrading ~s", [rebar_app_info:name(AppInfo)]),
             case rebar_fetch:download_source(AppDir, Source, State) of
-                {error, Reason} ->
-                    throw(Reason);
-                Result ->
-                    Result
+                true ->
+                    true;
+                Error ->
+                    throw(Error)
             end;
         false ->
             ?INFO("No upgrade needed for ~s", [rebar_app_info:name(AppInfo)]),
@@ -610,7 +613,7 @@ parse_goal(Name, Constraint) ->
 
 warn_skip_deps(AppInfo, State) ->
     Msg = "Skipping ~s (from ~p) as an app of the same name "
-          "has already been fetched~n",
+          "has already been fetched",
     Args = [rebar_app_info:name(AppInfo),
             rebar_app_info:source(AppInfo)],
     case rebar_state:get(State, deps_error_on_conflict, false) of
@@ -620,7 +623,7 @@ warn_skip_deps(AppInfo, State) ->
 
 warn_skip_pkg({Name, Source}, State) ->
     Msg = "Skipping ~s (version ~s from package index) as an app of the same "
-          "name has already been fetched~n",
+          "name has already been fetched",
     Args = [Name, Source],
     case rebar_state:get(State, deps_error_on_conflict, false) of
         false -> ?WARN(Msg, Args);

src/rebar_prv_update.erl

@@ -35,8 +35,7 @@ init(State) ->
 do(State) ->
     ?INFO("Updating package index...", []),
     try
-        Dir = rebar_dir:global_cache_dir(State),
-        RegistryDir = filename:join(Dir, "packages"),
+        RegistryDir = rebar_packages:package_dir(State),
         filelib:ensure_dir(filename:join(RegistryDir, "dummy")),
         HexFile = filename:join(RegistryDir, "registry"),
         TmpDir = ec_file:insecure_mkdtemp(),
@@ -64,8 +63,7 @@ format_error(package_index_write) ->
     "Failed to write package index.".
 
 write_registry(Dict, {digraph, Edges, Vertices, Neighbors, _}, State) ->
-    Dir = rebar_dir:global_cache_dir(State),
-    RegistryDir = filename:join(Dir, "packages"),
+    RegistryDir = rebar_packages:package_dir(State),
     filelib:ensure_dir(filename:join(RegistryDir, "dummy")),
     ets:tab2file(Edges, filename:join(RegistryDir, "edges")),
     ets:tab2file(Vertices, filename:join(RegistryDir, "vertices")),

src/rebar_resource.erl

@@ -14,23 +14,6 @@
 -type location() :: string().
 -type ref() :: any().
 
--ifdef(no_callback_support).
-
-%% In the case where R14 or lower is being used to compile the system
-%% we need to export a behaviour info
--export([behaviour_info/1]).
-
--spec behaviour_info(atom()) -> [{atom(), arity()}] | undefined.
-behaviour_info(callbacks) ->
-    [{lock, 2},
-     {download, 3},
-     {needs_update,2},
-     {make_vsn, 1}];
-behaviour_info(_) ->
-    undefined.
-
--else.
-
 -callback lock(file:filename_all(), tuple()) ->
     rebar_resource:resource().
 -callback download(file:filename_all(), tuple(), rebar_state:t()) ->
@@ -39,5 +22,3 @@ behaviour_info(_) ->
     boolean().
 -callback make_vsn(file:filename_all()) ->
     {plain, string()} | {error, string()}.
-
--endif.

test/mock_pkg_resource.erl

@@ -16,6 +16,7 @@ mock() -> mock([]).
 -spec mock(Opts) -> ok when
     Opts :: [Option],
     Option :: {update, [App]}
+            | {cache_dir, string()}
             | {default_vsn, Vsn}
             | {override_vsn, [{App, Vsn}]}
             | {not_in_index, [{App, Vsn}]}
@@ -83,7 +84,8 @@ mock_download(Opts) ->
                 [kernel, stdlib] ++ [element(1,D) || D  <- AppDeps]
             ),
             rebar_test_utils:create_config(Dir, [{deps, AppDeps}]),
-            Tarball = filename:join([Dir, App++"-"++binary_to_list(Vsn)++".tar"]),
+            TarApp = App++"-"++binary_to_list(Vsn)++".tar",
+            Tarball = filename:join([Dir, TarApp]),
             Contents = filename:join([Dir, "contents.tar.gz"]),
             Files = all_files(rebar_app_info:dir(AppInfo)),
             ok = erl_tar:create(Contents,
@@ -92,8 +94,11 @@ mock_download(Opts) ->
             ok = erl_tar:create(Tarball,
                                 [{"contents.tar.gz", Contents}],
                                 []),
-            [file:delete(F) || F <- Files],
-            {tarball, Tarball}
+            Cache = proplists:get_value(cache_dir, Opts, filename:join(Dir,"cache")),
+            Cached = filename:join([Cache, TarApp]),
+            filelib:ensure_dir(Cached),
+            rebar_file_utils:mv(Tarball, Cached),
+            {ok, true}
         end).
 
 %% @doc On top of the pkg resource mocking, we need to mock the package

test/rebar_pkg_SUITE.erl

@@ -0,0 +1,207 @@
+%% Test suite for the rebar pkg index caching and decompression
+%% mechanisms.
+-module(rebar_pkg_SUITE).
+-compile(export_all).
+-include_lib("common_test/include/ct.hrl").
+-include_lib("eunit/include/eunit.hrl").
+
+-define(bad_etag, "abcdef").
+-define(good_etag, "22e1d7387c9085a462340088a2a8ba67").
+-define(bad_checksum, <<"D576B442A68C7B92BACDE1EFE9C6E54D8D6C74BDB71D8175B9D3C6EC8C7B62A7">>).
+-define(good_checksum, <<"1C6CE379D191FBAB41B7905075E0BF87CBBE23C77CECE775C5A0B786B2244C35">>).
+
+all() -> [good_uncached, good_cached, badindexchk, badpkg,
+          bad_to_good, good_disconnect, bad_disconnect].
+
+init_per_suite(Config) ->
+    application:start(meck),
+    Config.
+
+end_per_suite(_Config) ->
+    application:stop(meck).
+
+init_per_testcase(good_uncached=Name, Config0) ->
+    Config = [{good_cache, false},
+              {pkg, {<<"goodpkg">>, <<"1.0.0">>}}
+             | Config0],
+    mock_config(Name, Config);
+init_per_testcase(good_cached=Name, Config0) ->
+    Pkg = {<<"goodpkg">>, <<"1.0.0">>},
+    Config1 = [{good_cache, true},
+               {pkg, Pkg}
+              | Config0],
+    Config = mock_config(Name, Config1),
+    copy_to_cache(Pkg, Config),
+    Config;
+init_per_testcase(badindexchk=Name, Config0) ->
+    Config = [{good_cache, false},
+              {pkg, {<<"badindexchk">>, <<"1.0.0">>}}
+             | Config0],
+    mock_config(Name, Config);
+init_per_testcase(badpkg=Name, Config0) ->
+    Config = [{good_cache, false},
+              {pkg, {<<"badpkg">>, <<"1.0.0">>}}
+             | Config0],
+    mock_config(Name, Config);
+init_per_testcase(bad_to_good=Name, Config0) ->
+    Config1 = [{good_cache, false},
+              {pkg, {<<"goodpkg">>, <<"1.0.0">>}}
+              | Config0],
+    Config = mock_config(Name, Config1),
+    Source = filename:join(?config(data_dir, Config), <<"badpkg-1.0.0.tar">>),
+    Dest = filename:join(?config(cache_dir, Config), <<"goodpkg-1.0.0.tar">>),
+    ec_file:copy(Source, Dest),
+    Config;
+init_per_testcase(good_disconnect=Name, Config0) ->
+    Pkg = {<<"goodpkg">>, <<"1.0.0">>},
+    Config1 = [{good_cache, true},
+               {pkg, Pkg}
+              | Config0],
+    Config = mock_config(Name, Config1),
+    copy_to_cache(Pkg, Config),
+    meck:unload(httpc),
+    meck:new(httpc, [passthrough, unsticky]),
+    meck:expect(httpc, request, fun(_, _, _, _) -> {error, econnrefused} end),
+    Config;
+init_per_testcase(bad_disconnect=Name, Config0) ->
+    Pkg = {<<"goodpkg">>, <<"1.0.0">>},
+    Config1 = [{good_cache, false},
+               {pkg, Pkg}
+              | Config0],
+    Config = mock_config(Name, Config1),
+    meck:unload(httpc),
+    meck:new(httpc, [passthrough, unsticky]),
+    meck:expect(httpc, request, fun(_, _, _, _) -> {error, econnrefused} end),
+    Config.
+
+end_per_testcase(_, Config) ->
+    unmock_config(Config),
+    Config.
+
+good_uncached(Config) ->
+    Tmp = ?config(tmp_dir, Config),
+    {Pkg,Vsn} = ?config(pkg, Config),
+    State = ?config(state, Config),
+    ?assertEqual({ok, true},
+                 rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn}, State)),
+    Cache = ?config(cache_dir, Config),
+    ?assert(filelib:is_regular(filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".tar">>))).
+
+good_cached(Config) ->
+    Tmp = ?config(tmp_dir, Config),
+    {Pkg,Vsn} = ?config(pkg, Config),
+    State = ?config(state, Config),
+    Cache = ?config(cache_dir, Config),
+    CachedFile = filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".tar">>),
+    ?assert(filelib:is_regular(CachedFile)),
+    {ok, Content} = file:read_file(CachedFile),
+    ?assertEqual({ok, true},
+                 rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn}, State)),
+    {ok, Content} = file:read_file(CachedFile).
+
+badindexchk(Config) ->
+    Tmp = ?config(tmp_dir, Config),
+    {Pkg,Vsn} = ?config(pkg, Config),
+    State = ?config(state, Config),
+    ?assertMatch({bad_registry_checksum, _Path},
+                 rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn}, State)),
+    %% The cached file is there for forensic purposes
+    Cache = ?config(cache_dir, Config),
+    ?assert(filelib:is_regular(filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".tar">>))).
+
+badpkg(Config) ->
+    Tmp = ?config(tmp_dir, Config),
+    {Pkg,Vsn} = ?config(pkg, Config),
+    State = ?config(state, Config),
+    ?assertMatch({bad_download, _Path},
+                 rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn}, State)),
+    %% The cached file is there for forensic purposes
+    Cache = ?config(cache_dir, Config),
+    ?assert(filelib:is_regular(filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".tar">>))).
+
+bad_to_good(Config) ->
+    Tmp = ?config(tmp_dir, Config),
+    {Pkg,Vsn} = ?config(pkg, Config),
+    State = ?config(state, Config),
+    Cache = ?config(cache_dir, Config),
+    CachedFile = filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".tar">>),
+    ?assert(filelib:is_regular(CachedFile)),
+    {ok, Contents} = file:read_file(CachedFile),
+    ?assertEqual({ok, true},
+                 rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn}, State)),
+    %% Cache has refreshed
+    ?assert({ok, Contents} =/= file:read_file(CachedFile)).
+
+good_disconnect(Config) ->
+    Tmp = ?config(tmp_dir, Config),
+    {Pkg,Vsn} = ?config(pkg, Config),
+    State = ?config(state, Config),
+    Cache = ?config(cache_dir, Config),
+    CachedFile = filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".tar">>),
+    ?assert(filelib:is_regular(CachedFile)),
+    {ok, Content} = file:read_file(CachedFile),
+    ?assertEqual({ok, true},
+                 rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn}, State)),
+    {ok, Content} = file:read_file(CachedFile).
+
+bad_disconnect(Config) ->
+    Tmp = ?config(tmp_dir, Config),
+    {Pkg,Vsn} = ?config(pkg, Config),
+    State = ?config(state, Config),
+    ?assertEqual(request_failed,
+                 rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn}, State)).
+
+
+%%%%%%%%%%%%%%%
+%%% Helpers %%%
+%%%%%%%%%%%%%%%
+mock_config(Name, Config) ->
+    Priv = ?config(priv_dir, Config),
+    CacheRoot = filename:join([Priv, "cache", atom_to_list(Name)]),
+    TmpDir = filename:join([Priv, "tmp", atom_to_list(Name)]),
+    T = ets:new(fake_registry, [public]),
+    ets:insert_new(T, [
+        {{<<"badindexchk">>,<<"1.0.0">>}, [[], ?bad_checksum]},
+        {{<<"goodpkg">>,<<"1.0.0">>}, [[], ?good_checksum]},
+        {{<<"badpkg">>,<<"1.0.0">>}, [[], ?good_checksum]}
+    ]),
+    CacheDir = filename:join([CacheRoot, "hex", "com", "test", "packages"]),
+    filelib:ensure_dir(filename:join([CacheDir, "registry"])),
+    ok = ets:tab2file(T, filename:join([CacheDir, "registry"])),
+    %% The state returns us a fake registry
+    meck:new(rebar_state, [passthrough]),
+    meck:expect(rebar_state, registry,
+                fun(_State) -> {ok, fake_registry} end),
+    meck:expect(rebar_state, get,
+                fun(_State, rebar_packages_cdn, _Default) ->
+                    "http://test.com/"
+                end),
+    meck:new(rebar_dir, [passthrough]),
+    meck:expect(rebar_dir, global_cache_dir, fun(_) -> CacheRoot end),
+    %% Cache fetches are mocked -- we assume the server and clients are
+    %% correctly used.
+    GoodCache = ?config(good_cache, Config),
+    {Pkg,Vsn} = ?config(pkg, Config),
+    PkgFile = <<Pkg/binary, "-", Vsn/binary, ".tar">>,
+    {ok, PkgContents} = file:read_file(filename:join(?config(data_dir, Config), PkgFile)),
+    meck:new(httpc, [passthrough, unsticky]),
+    meck:expect(httpc, request,
+            fun(get, {_Url, _Opts}, _, _) when GoodCache ->
+                {ok, {{Vsn, 304, <<"Not Modified">>}, [{"etag", ?good_etag}], <<>>}};
+               (get, {_Url, _Opts}, _, _) ->
+                {ok, {{Vsn, 200, <<"OK">>}, [{"etag", ?good_etag}], PkgContents}}
+            end),
+    [{cache_root, CacheRoot},
+     {cache_dir, CacheDir},
+     {tmp_dir, TmpDir},
+     {mock_table, T} | Config].
+
+unmock_config(Config) ->
+    meck:unload(),
+    ets:delete(?config(mock_table, Config)).
+
+copy_to_cache({Pkg,Vsn}, Config) ->
+    Name = <<Pkg/binary, "-", Vsn/binary, ".tar">>,
+    Source = filename:join(?config(data_dir, Config), Name),
+    Dest = filename:join(?config(cache_dir, Config), Name),
+    ec_file:copy(Source, Dest).