store hex package downloads in shared cache dir

rebar.config

@@ -30,9 +30,7 @@
                       {"rebar/include/*", "."}]}.
 
 {erl_opts,
- [{platform_define, "R14", no_callback_support},
-  {platform_define, "^[0-9]+", namespaced_types},
-  {platform_define, "^R1[4|5]", deprecated_crypto},
+ [{platform_define, "^[0-9]+", namespaced_types},
   no_debug_info,
   warnings_as_errors]}.
 

src/rebar_fetch.erl

@@ -40,20 +40,15 @@ download_source(AppDir, Source, State) ->
                 ok = rebar_file_utils:mv(TmpDir, filename:absname(AppDir1)),
                 true;
             {tarball, File} ->
-                Contents = filename:join(TmpDir, "contents"),
                 ec_file:mkdir_p(AppDir1),
-                ec_file:mkdir_p(Contents),
-                ok = erl_tar:extract(File, [{cwd, TmpDir}]),
-                ok = erl_tar:extract(filename:join(TmpDir, "contents.tar.gz"),
-                                    [{cwd, Contents}, compressed]),
+                {ok, Files} = erl_tar:extract(File, [memory]),
+
                 code:del_path(filename:absname(filename:join(AppDir1, "ebin"))),
                 ec_file:remove(filename:absname(AppDir1), [recursive]),
 
-                ?DEBUG("Moving contents ~p to ~p", [Contents, filename:absname(AppDir1)]),
-                ok = rebar_file_utils:mv(Contents, filename:absname(AppDir1)),
-
-                ?DEBUG("Removing tmp dir ~p", [TmpDir]),
-                ec_file:remove(TmpDir, [recursive]),
+                {"contents.tar.gz", Binary} = lists:keyfind("contents.tar.gz", 1, Files),
+                ok = erl_tar:extract({binary, Binary},
+                                     [{cwd, filename:absname(AppDir1)}, compressed]),
                 true
         end
     catch

src/rebar_pkg_resource.erl

@@ -9,8 +9,11 @@
         ,needs_update/2
         ,make_vsn/1]).
 
+-include_lib("providers/include/providers.hrl").
 -include("rebar.hrl").
 
+-define(DEFAULT_CDN, "https://s3.amazonaws.com/s3.hex.pm/tarballs").
+
 lock(_AppDir, Source) ->
     Source.
 
@@ -23,12 +26,67 @@ needs_update(Dir, {pkg, _Name, Vsn}) ->
             true
     end.
 
-download(Dir, {pkg, Name, Vsn}, State) ->
-    TmpFile = filename:join(Dir, "package.tar.gz"),
-    CDN = rebar_state:get(State, rebar_packages_cdn, "https://s3.amazonaws.com/s3.hex.pm/tarballs"),
-    Url = string:join([CDN, binary_to_list(<<Name/binary, "-", Vsn/binary, ".tar">>)], "/"),
-    {ok, saved_to_file} = httpc:request(get, {Url, []}, [], [{stream, TmpFile}]),
-    {tarball, TmpFile}.
+download(_Dir, {pkg, Name, Vsn}, State) ->
+    CDN = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN),
+    PackageDir = hex_package_dir(CDN, State),
+
+    Package = binary_to_list(<<Name/binary, "-", Vsn/binary, ".tar">>),
+    Path = filename:join(PackageDir, Package),
+    Url = string:join([CDN, Package], "/"),
+
+    case request(Url, etag(Path)) of
+        {ok, cached} ->
+            {tarball, Path};
+        {ok, Binary} ->
+            file:write_file(Path, Binary),
+            {tarball, Path};
+        error ->
+            case filelib:is_regular(Path) of
+                true ->
+                    ?DEBUG("Download ~s error, using ~s since it exists", [Url, Path]),
+                    {tarball, Path};
+                false ->
+                    throw(request_failed)
+            end
+    end.
 
 make_vsn(_) ->
     {error, "Replacing version of type pkg not supported."}.
+
+%% Use the shared hex package directory unless a non-default package repo is used
+hex_package_dir(?DEFAULT_CDN, _) ->
+    filename:join([rebar_dir:home_dir(), ".hex", "packages"]);
+hex_package_dir(CDN, State) ->
+    CacheDir = rebar_dir:global_cache_dir(State),
+    {ok, {_, _, Host, _, _, _}} = http_uri:parse(CDN),
+    CDNPath = filename:join(lists:reverse(string:tokens(Host, "."))),
+    PackageDir = filename:join([CacheDir, "hex", CDNPath, "packages"]),
+    ok = filelib:ensure_dir(filename:join(PackageDir, "placeholder")),
+    PackageDir.
+
+request(Url, ETag) ->
+    case httpc:request(get, {Url, [{"if-none-match", ETag} || ETag =/= false]},
+                      [{relaxed, true}],
+                      [{body_format, binary}]) of
+        {ok, {{_Version, 200, _Reason}, _Headers, Body}} ->
+            ?DEBUG("Successfully downloaded ~s", [Url]),
+            {ok, Body};
+        {ok, {{_Version, 304, _Reason}, _Headers, _Body}} ->
+            ?DEBUG("Cached copy of ~s still valid", [Url]),
+            {ok, cached};
+        {ok, {{_Version, Code, _Reason}, _Headers, _Body}} ->
+            ?DEBUG("Request to ~p failed: status code ~p", [Url, Code]),
+            error;
+        {error, Reason} ->
+            ?DEBUG("Request to ~p failed: ~p", [Url, Reason]),
+            error
+    end.
+
+etag(Path) ->
+    case file:read_file(Path) of
+        {ok, Binary} ->
+            <<X:128/big-unsigned-integer>> = crypto:hash(md5, Binary),
+            string:to_lower(lists:flatten(io_lib:format("~32.16.0b", [X])));
+        {error, _} ->
+            false
+    end.

src/rebar_resource.erl

@@ -14,23 +14,6 @@
 -type location() :: string().
 -type ref() :: any().
 
--ifdef(no_callback_support).
-
-%% In the case where R14 or lower is being used to compile the system
-%% we need to export a behaviour info
--export([behaviour_info/1]).
-
--spec behaviour_info(atom()) -> [{atom(), arity()}] | undefined.
-behaviour_info(callbacks) ->
-    [{lock, 2},
-     {download, 3},
-     {needs_update,2},
-     {make_vsn, 1}];
-behaviour_info(_) ->
-    undefined.
-
--else.
-
 -callback lock(file:filename_all(), tuple()) ->
     rebar_resource:resource().
 -callback download(file:filename_all(), tuple(), rebar_state:t()) ->
@@ -39,5 +22,3 @@ behaviour_info(_) ->
     boolean().
 -callback make_vsn(file:filename_all()) ->
     {plain, string()} | {error, string()}.
-
--endif.