Ad-hoc attempt at restructuring pkg cache

src/rebar_fetch.erl

@@ -31,20 +31,14 @@ download_source(AppDir, Source, State) ->
         Module = get_resource_type(Source, Resources),
         TmpDir = ec_file:insecure_mkdtemp(),
         AppDir1 = ec_cnv:to_list(AppDir),
-        case Module:download(TmpDir, Source, State) of
-            {ok, _} ->
-                ec_file:mkdir_p(AppDir1),
-                code:del_path(filename:absname(filename:join(AppDir1, "ebin"))),
-                ec_file:remove(filename:absname(AppDir1), [recursive]),
-                ?DEBUG("Moving checkout ~p to ~p", [TmpDir, filename:absname(AppDir1)]),
-                ok = rebar_file_utils:mv(TmpDir, filename:absname(AppDir1)),
-                true;
-            {tarball, File} ->
-                verify_and_extract(File, Source, AppDir1, State)
-        end
+        {ok, _} = Module:download(TmpDir, Source, State),
+        ec_file:mkdir_p(AppDir1),
+        code:del_path(filename:absname(filename:join(AppDir1, "ebin"))),
+        ec_file:remove(filename:absname(AppDir1), [recursive]),
+        ?DEBUG("Moving checkout ~p to ~p", [TmpDir, filename:absname(AppDir1)]),
+        ok = rebar_file_utils:mv(TmpDir, filename:absname(AppDir1)),
+        true
     catch
-        _:bad_etag ->
-            throw(?PRV_ERROR({bad_etag, Source}));
         C:T ->
             ?DEBUG("rebar_fetch exception ~p ~p ~p", [C, T, erlang:get_stacktrace()]),
             throw(?PRV_ERROR({fetch_fail, Source}))
@@ -92,34 +86,3 @@ find_resource_module(Type, Location, Resources) ->
         {Type, Module} ->
             Module
     end.
-
-verify_and_extract(File, Source, AppDir, State) ->
-    ec_file:mkdir_p(AppDir),
-    {ok, Files} = erl_tar:extract(File, [memory]),
-
-    code:del_path(filename:absname(filename:join(AppDir, "ebin"))),
-    ec_file:remove(filename:absname(AppDir), [recursive]),
-
-    {"contents.tar.gz", Contents} = lists:keyfind("contents.tar.gz", 1, Files),
-    {"VERSION", Version} = lists:keyfind("VERSION", 1, Files),
-    {"metadata.config", Meta} = lists:keyfind("metadata.config", 1, Files),
-
-    Checksum = checksum(Contents, Version, Meta),
-    RegistryChecksum = rebar_packages:registry_checksum(Source, State),
-    {"CHECKSUM", TarChecksum} = lists:keyfind("CHECKSUM", 1, Files),
-
-    if
-        Checksum =/= TarChecksum ->
-            ?PRV_ERROR({bad_checksum, File});
-        Checksum =/= RegistryChecksum ->
-            ?PRV_ERROR({bad_registry_checksum, File});
-        true ->
-            ok = erl_tar:extract({binary, Contents},
-                                 [{cwd, filename:absname(AppDir)}, compressed]),
-            true
-    end.
-
-checksum(Contents, Version, Meta) ->
-    Blob = <<Version/binary, Meta/binary, Contents/binary>>,
-    <<X:256/big-unsigned-integer>> = crypto:hash(sha256, Blob),
-    list_to_binary(string:to_upper(lists:flatten(io_lib:format("~64.16.0b", [X])))).

src/rebar_pkg_resource.erl

@@ -26,38 +26,69 @@ needs_update(Dir, {pkg, _Name, Vsn}) ->
             true
     end.
 
-download(_Dir, {pkg, Name, Vsn}, State) ->
+download(TmpDir, Pkg={pkg, Name, Vsn}, State) ->
     CDN = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN),
     PackageDir = hex_package_dir(CDN, State),
-
     Package = binary_to_list(<<Name/binary, "-", Vsn/binary, ".tar">>),
-    Path = filename:join(PackageDir, Package),
+    CachePath = filename:join(PackageDir, Package),
     Url = string:join([CDN, Package], "/"),
+    cached_download(TmpDir, CachePath, Pkg, Url, etag(CachePath), State).
 
-    case request(Url, etag(Path)) of
+cached_download(TmpDir, CachePath, Pkg, Url, ETag, State) ->
+    case request(Url, ETag) of
         {ok, cached} ->
-            {tarball, Path};
-        {ok, Binary, EtagHeader} ->
-            file:write_file(Path, Binary),
-            Etag = etag(Path),
-            case EtagHeader =:= Etag of
-                true ->
-                    {tarball, Path};
-                false ->
-                    ?DEBUG("Bad md5sum for ~s of ~s comparing to ~s sent by server",
-                           [Path, Etag, EtagHeader]),
-                    throw(bad_etag)
-            end;
+            serve_from_cache(TmpDir, CachePath, Pkg, State);
+        {ok, Body, NewETag} ->
+            serve_from_download(TmpDir, CachePath, Pkg, NewETag, Body, State);
+        error when ETag =/= false ->
+            ?DEBUG("Download ~s error, using ~s from cache", [Url, CachePath]),
+            serve_from_cache(TmpDir, CachePath, Pkg, State);
         error ->
-            case filelib:is_regular(Path) of
-                true ->
-                    ?DEBUG("Download ~s error, using ~s since it exists", [Url, Path]),
-                    {tarball, Path};
-                false ->
-                    throw(request_failed)
-            end
+            throw(request_failed)
+    end.
+
+serve_from_cache(TmpDir, CachePath, Pkg, State) ->
+    {Files, Contents, Version, Meta} = extract(TmpDir, CachePath),
+    case checksums(Pkg, Files, Contents, Version, Meta, State) of
+        {Chk, Chk, Chk} ->
+            ok = erl_tar:extract({binary, Contents}, [{cwd, TmpDir}, compressed]),
+            {ok, true};
+        {_Bin, Chk, Chk} ->
+            ?PRV_ERROR({failed_extract, CachePath});
+        {Chk, _Reg, Chk} ->
+            ?PRV_ERROR({bad_registry_checksum, CachePath});
+        {_Bin, _Reg, _Tar} ->
+            ?PRV_ERROR({bad_checksum, CachePath})
+    end.
+
+serve_from_download(TmpDir, CachePath, Package, ETag, Binary, State) ->
+    ?DEBUG("Writing ~p to cache at ~s", [Package, CachePath]),
+    file:write_file(CachePath, Binary),
+    case etag(CachePath) of
+        ETag ->
+            serve_from_cache(TmpDir, CachePath, Package, State);
+        FileETag ->
+            ?DEBUG("Download ETag ~s doesn't match cached ETag ~s", [ETag, FileETag]),
+            ?PRV_ERROR({bad_download, CachePath})
     end.
 
+
+extract(TmpDir, CachePath) ->
+    ec_file:mkdir_p(TmpDir),
+    {ok, Files} = erl_tar:extract(CachePath, [memory]),
+    {"contents.tar.gz", Contents} = lists:keyfind("contents.tar.gz", 1, Files),
+    {"VERSION", Version} = lists:keyfind("VERSION", 1, Files),
+    {"metadata.config", Meta} = lists:keyfind("metadata.config", 1, Files),
+    {Files, Contents, Version, Meta}.
+
+checksums(Pkg, Files, Contents, Version, Meta, State) ->
+    Blob = <<Version/binary, Meta/binary, Contents/binary>>,
+    <<X:256/big-unsigned>> = crypto:hash(sha256, Blob),
+    BinChecksum = list_to_binary(string:to_upper(lists:flatten(io_lib:format("~64.16.0b", [X])))),
+    RegistryChecksum = rebar_packages:registry_sum(Pkg, State),
+    {"CHECKSUM", TarChecksum} = lists:keyfind("CHECKSUM", 1, Files),
+    {BinChecksum, RegistryChecksum, TarChecksum}.
+
 make_vsn(_) ->
     {error, "Replacing version of type pkg not supported."}.
 
@@ -77,8 +108,8 @@ request(Url, ETag) ->
                       [{relaxed, true}],
                       [{body_format, binary}]) of
         {ok, {{_Version, 200, _Reason}, Headers, Body}} ->
-            {"etag", ETag1} = lists:keyfind("etag", 1, Headers),
             ?DEBUG("Successfully downloaded ~s", [Url]),
+            {"etag", ETag1} = lists:keyfind("etag", 1, Headers),
             {ok, Body, string:strip(ETag1, both, $")};
         {ok, {{_Version, 304, _Reason}, _Headers, _Body}} ->
             ?DEBUG("Cached copy of ~s still valid", [Url]),