support temporary cdn change with HEX_CDN os var

.gitignore

@@ -1,3 +1,4 @@
+_checkouts
 .rebar3
 rebar3
 _build

src/rebar.hrl

@@ -22,8 +22,9 @@
 -define(DEFAULT_TEST_DEPS_DIR, "test/lib").
 -define(DEFAULT_RELEASE_DIR, "rel").
 -define(DEFAULT_CONFIG_FILE, "rebar.config").
--define(DEFAULT_CDN, "https://s3.amazonaws.com/s3.hex.pm/tarballs").
--define(DEFAULT_HEX_REGISTRY, "https://s3.amazonaws.com/s3.hex.pm/registry.ets.gz").
+-define(DEFAULT_CDN, "https://s3.amazonaws.com/s3.hex.pm/").
+-define(REMOTE_PACKAGE_DIR, "tarballs").
+-define(REMOTE_REGISTRY_FILE, "registry.ets.gz").
 -define(LOCK_FILE, "rebar.lock").
 
 -define(PACKAGE_INDEX_VERSION, 3).

src/rebar3.erl

@@ -105,25 +105,32 @@ run_aux(State, RawArgs) ->
                      rebar_state:apply_profiles(State, [list_to_atom(Profile)])
              end,
 
+    State2 = case os:getenv("HEX_CDN") of
+                 false ->
+                     State1;
+                 CDN ->
+                     rebar_state:set(State1, rebar_packages_cdn, CDN)
+             end,
+
     %% bootstrap test profile
-    State2 = rebar_state:add_to_profile(State1, test, test_state(State1)),
+    State3 = rebar_state:add_to_profile(State2, test, test_state(State1)),
 
     %% Process each command, resetting any state between each one
     BaseDir = rebar_state:get(State, base_dir, ?DEFAULT_BASE_DIR),
-    State3 = rebar_state:set(State2, base_dir,
-                             filename:join(filename:absname(rebar_state:dir(State2)), BaseDir)),
+    State4 = rebar_state:set(State3, base_dir,
+                             filename:join(filename:absname(rebar_state:dir(State3)), BaseDir)),
 
     {ok, Providers} = application:get_env(rebar, providers),
     %% Providers can modify profiles stored in opts, so set default after initializing providers
-    State4 = rebar_state:create_logic_providers(Providers, State3),
-    State5 = rebar_plugins:project_apps_install(State4),
-    State6 = rebar_state:default(State5, rebar_state:opts(State5)),
+    State5 = rebar_state:create_logic_providers(Providers, State4),
+    State6 = rebar_plugins:project_apps_install(State5),
+    State7 = rebar_state:default(State6, rebar_state:opts(State6)),
 
     {Task, Args} = parse_args(RawArgs),
 
-    State7 = rebar_state:code_paths(State6, default, code:get_path()),
+    State8 = rebar_state:code_paths(State7, default, code:get_path()),
 
-    rebar_core:init_command(rebar_state:command_args(State7, Args), Task).
+    rebar_core:init_command(rebar_state:command_args(State8, Args), Task).
 
 init_config() ->
     %% Initialize logging system
@@ -339,4 +346,4 @@ safe_define_test_macro(Opts) ->
 test_defined([{d, 'TEST'}|_]) -> true;
 test_defined([{d, 'TEST', true}|_]) -> true;
 test_defined([_|Rest]) -> test_defined(Rest);
-test_defined([]) -> false.
+test_defined([]) -> false.

src/rebar_packages.erl

@@ -46,7 +46,7 @@ close_packages() ->
     catch ets:delete(?PACKAGE_TABLE).
 
 load_and_verify_version(State) ->
-    RegistryDir = registry_dir(State),
+    {ok, RegistryDir} = registry_dir(State),
     case ets:file2tab(filename:join(RegistryDir, ?INDEX_FILE)) of
         {ok, _} ->
             case ets:lookup_element(?PACKAGE_TABLE, package_index_version, 2) of
@@ -89,21 +89,30 @@ registry_dir(State) ->
         ?DEFAULT_CDN ->
             RegistryDir = filename:join([CacheDir, "hex", "default"]),
             ok = filelib:ensure_dir(filename:join(RegistryDir, "placeholder")),
-            RegistryDir;
+            {ok, RegistryDir};
         CDN ->
-            {ok, {_, _, Host, _, Path, _}} = http_uri:parse(CDN),
-            CDNHostPath = lists:reverse(string:tokens(Host, ".")),
-            CDNPath = tl(filename:split(Path)),
-            RegistryDir = filename:join([CacheDir, "hex"] ++ CDNHostPath ++ CDNPath),
-            ok = filelib:ensure_dir(filename:join(RegistryDir, "placeholder")),
-            RegistryDir
+            case rebar_utils:url_append_path(CDN, ?REMOTE_PACKAGE_DIR) of
+                {ok, Parsed} ->
+                    {ok, {_, _, Host, _, Path, _}} = http_uri:parse(Parsed),
+                    CDNHostPath = lists:reverse(string:tokens(Host, ".")),
+                    CDNPath = tl(filename:split(Path)),
+                    RegistryDir = filename:join([CacheDir, "hex"] ++ CDNHostPath ++ CDNPath),
+                    ok = filelib:ensure_dir(filename:join(RegistryDir, "placeholder")),
+                    {ok, RegistryDir};
+                _ ->
+                    {uri_parse_error, CDN}
+            end
     end.
 
 package_dir(State) ->
-    RegistryDir = registry_dir(State),
-    PackageDir = filename:join([RegistryDir, "packages"]),
-    ok = filelib:ensure_dir(filename:join(PackageDir, "placeholder")),
-    PackageDir.
+    case registry_dir(State) of
+        {ok, RegistryDir} ->
+            PackageDir = filename:join([RegistryDir, "packages"]),
+            ok = filelib:ensure_dir(filename:join(PackageDir, "placeholder")),
+            {ok, PackageDir};
+        Error ->
+            Error
+    end.
 
 registry_checksum({pkg, Name, Vsn}, State) ->
     try

src/rebar_pkg_resource.erl

@@ -30,11 +30,15 @@ needs_update(Dir, {pkg, _Name, Vsn}) ->
 
 download(TmpDir, Pkg={pkg, Name, Vsn}, State) ->
     CDN = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN),
-    PackageDir = rebar_packages:package_dir(State),
+    {ok, PackageDir} = rebar_packages:package_dir(State),
     Package = binary_to_list(<<Name/binary, "-", Vsn/binary, ".tar">>),
     CachePath = filename:join(PackageDir, Package),
-    Url = string:join([CDN, Package], "/"),
-    cached_download(TmpDir, CachePath, Pkg, Url, etag(CachePath), State).
+    case rebar_utils:url_append_path(CDN, filename:join(?REMOTE_PACKAGE_DIR, Package)) of
+        {ok, Url} ->
+            cached_download(TmpDir, CachePath, Pkg, Url, etag(CachePath), State);
+        _ ->
+            {fetch_fail, Name, Vsn}
+    end.
 
 cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn}, Url, ETag, State) ->
     case request(Url, ETag) of

src/rebar_prv_update.erl

@@ -36,26 +36,36 @@ init(State) ->
 -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}.
 do(State) ->
     try
-        RegistryDir = rebar_packages:registry_dir(State),
-        filelib:ensure_dir(filename:join(RegistryDir, "dummy")),
-        HexFile = filename:join(RegistryDir, "registry"),
-        ?INFO("Updating package registry...", []),
-        TmpDir = ec_file:insecure_mkdtemp(),
-        TmpFile = filename:join(TmpDir, "packages.gz"),
+        case rebar_packages:registry_dir(State) of
+            {ok, RegistryDir} ->
+                filelib:ensure_dir(filename:join(RegistryDir, "dummy")),
+                HexFile = filename:join(RegistryDir, "registry"),
+                ?INFO("Updating package registry...", []),
+                TmpDir = ec_file:insecure_mkdtemp(),
+                TmpFile = filename:join(TmpDir, "packages.gz"),
 
-        Url = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_HEX_REGISTRY),
-        case httpc:request(get, {Url, []},
-                           [], [{stream, TmpFile}, {sync, true}],
-                           rebar) of
-            {ok, saved_to_file} ->
-                {ok, Data} = file:read_file(TmpFile),
-                Unzipped = zlib:gunzip(Data),
-                ok = file:write_file(HexFile, Unzipped),
-                ?INFO("Writing registry to ~s", [HexFile]),
-                hex_to_index(State),
-                {ok, State};
-            _ ->
-                ?PRV_ERROR(package_index_download)
+                CDN = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN),
+                case rebar_utils:url_append_path(CDN, ?REMOTE_REGISTRY_FILE) of
+                    {ok, Url} ->
+                        ?DEBUG("Fetching registry from ~p", [Url]),
+                        case httpc:request(get, {Url, []},
+                                           [], [{stream, TmpFile}, {sync, true}],
+                                           rebar) of
+                            {ok, saved_to_file} ->
+                                {ok, Data} = file:read_file(TmpFile),
+                                Unzipped = zlib:gunzip(Data),
+                                ok = file:write_file(HexFile, Unzipped),
+                                ?INFO("Writing registry to ~s", [HexFile]),
+                                hex_to_index(State),
+                                {ok, State};
+                            _ ->
+                                ?PRV_ERROR(package_index_download)
+                        end;
+                    _ ->
+                        ?PRV_ERROR({package_parse_cdn, CDN})
+                end;
+            {uri_parse_error, CDN} ->
+                ?PRV_ERROR({package_parse_cdn, CDN})
         end
     catch
         _E:C ->
@@ -64,6 +74,8 @@ do(State) ->
     end.
 
 -spec format_error(any()) -> iolist().
+format_error({package_parse_cdn, Uri}) ->
+    io_lib:format("Failed to parse CDN url: ~p", [Uri]);
 format_error(package_index_download) ->
     "Failed to download package index.";
 format_error(package_index_write) ->
@@ -75,7 +87,7 @@ is_supported(<<"rebar3">>) -> true;
 is_supported(_) -> false.
 
 hex_to_index(State) ->
-    RegistryDir = rebar_packages:registry_dir(State),
+    {ok, RegistryDir} = rebar_packages:registry_dir(State),
     HexFile = filename:join(RegistryDir, "registry"),
     try ets:file2tab(HexFile) of
         {ok, Registry} ->

src/rebar_utils.erl

@@ -61,6 +61,7 @@
          tup_find/2,
          line_count/1,
          set_httpc_options/0,
+         url_append_path/2,
          escape_chars/1,
          escape_double_quotes/1,
          escape_double_quotes_weak/1,
@@ -798,6 +799,15 @@ set_httpc_options(Scheme, Proxy) ->
     {ok, {_, _, Host, Port, _, _}} = http_uri:parse(Proxy),
     httpc:set_options([{Scheme, {{Host, Port}, []}}], rebar).
 
+url_append_path(Url, ExtraPath) ->
+     case http_uri:parse(Url) of
+         {ok, {Scheme, UserInfo, Host, Port, Path, Query}} ->
+             {ok, lists:append([atom_to_list(Scheme), "://", UserInfo, Host, ":", integer_to_list(Port),
+                                filename:join(Path, ExtraPath), "?", Query])};
+         _ ->
+             error
+     end.
+
 %% escape\ as\ a\ shell\?
 escape_chars(Str) when is_atom(Str) ->
     escape_chars(atom_to_list(Str));

test/rebar_pkg_SUITE.erl

@@ -30,8 +30,8 @@ init_per_testcase(pkgs_provider=Name, Config) ->
     filelib:ensure_dir(filename:join([CacheDir, "registry"])),
     ok = ets:tab2file(Tid, filename:join([CacheDir, "registry"])),
     meck:new(rebar_packages, [passthrough]),
-    meck:expect(rebar_packages, registry_dir, fun(_) -> CacheDir end),
-    meck:expect(rebar_packages, package_dir, fun(_) -> CacheDir end),
+    meck:expect(rebar_packages, registry_dir, fun(_) -> {ok, CacheDir} end),
+    meck:expect(rebar_packages, package_dir, fun(_) -> {ok, CacheDir} end),
     rebar_prv_update:hex_to_index(rebar_state:new()),
     Config;
 init_per_testcase(good_uncached=Name, Config0) ->
@@ -203,8 +203,8 @@ mock_config(Name, Config) ->
     meck:expect(rebar_dir, global_cache_dir, fun(_) -> CacheRoot end),
 
     meck:new(rebar_packages, [passthrough]),
-    meck:expect(rebar_packages, registry_dir, fun(_) -> CacheDir end),
-    meck:expect(rebar_packages, package_dir, fun(_) -> CacheDir end),
+    meck:expect(rebar_packages, registry_dir, fun(_) -> {ok, CacheDir} end),
+    meck:expect(rebar_packages, package_dir, fun(_) -> {ok, CacheDir} end),
     rebar_prv_update:hex_to_index(rebar_state:new()),
 
     %% Cache fetches are mocked -- we assume the server and clients are

test/rebar_pkg_alias_SUITE.erl

@@ -98,8 +98,8 @@ mock_config(Name, Config) ->
     meck:expect(rebar_dir, global_cache_dir, fun(_) -> CacheRoot end),
 
     meck:new(rebar_packages, [passthrough, no_link]),
-    meck:expect(rebar_packages, registry_dir, fun(_) -> CacheDir end),
-    meck:expect(rebar_packages, package_dir, fun(_) -> CacheDir end),
+    meck:expect(rebar_packages, registry_dir, fun(_) -> {ok, CacheDir} end),
+    meck:expect(rebar_packages, package_dir, fun(_) -> {ok, CacheDir} end),
     rebar_prv_update:hex_to_index(rebar_state:new()),
 
     %% Cache fetches are mocked -- we assume the server and clients are