add hex auth handling for repos (#1874)

auth token are kept in a hex.config file that is modified by the
rebar3 hex plugin.

Repo names that have a : separating a parent and child are considered
organizations. The parent repo's auth will be included with the child.
So an organization named hexpm:rebar3_test will include any hexpm
auth tokens found in the rebar3_test organization's configuration.

src/rebar.hrl

@@ -30,6 +30,8 @@
 -define(PACKAGE_INDEX_VERSION, 4).
 -define(PACKAGE_TABLE, package_index_v4).
 -define(INDEX_FILE, "packages-v4.idx").
+-define(HEX_AUTH_FILE, "hex.config").
+-define(PUBLIC_HEX_REPO, <<"hexpm">>).
 
 %% the package record is used in a select match spec which upsets dialyzer
 %% this is the suggested workaround from Tobias

src/rebar_hex_repos.erl

@@ -0,0 +1,122 @@
+-module(rebar_hex_repos).
+
+-export([from_state/2,
+         get_repo_config/2,
+         auth_config/1,
+         update_auth_config/2]).
+
+-ifdef(TEST).
+%% exported for test purposes
+-export([repos/1, merge_repos/1]).
+-endif.
+
+-include("rebar.hrl").
+
+-type repo() :: #{name => unicode:unicode_binary(),
+                  api_url => binary(),
+                  api_key => binary(),
+                  repo_url => binary(),
+                  repo_public_key => binary(),
+                  repo_verify => binary()}.
+
+from_state(BaseConfig, State) ->
+    HexConfig = rebar_state:get(State, hex, []),
+    Repos = repos(HexConfig),
+    %% auth is stored in a separate config file since the plugin generates and modifies it
+    Auth = ?MODULE:auth_config(State),
+    %% add base config entries that are specific to use by rebar3 and not overridable
+    Repos1 = merge_with_base_and_auth(Repos, BaseConfig, Auth),
+    %% merge organizations parent repo options into each oraganization repo
+    update_organizations(Repos1).
+
+-spec get_repo_config(unicode:unicode_binary(), rebar_state:t() | [hex_core:config()])
+                     -> {ok, hex_core:config()} | error.
+get_repo_config(RepoName, Repos) when is_list(Repos) ->
+    ec_lists:find(fun(#{name := N}) -> N =:= RepoName end, Repos);
+get_repo_config(RepoName, State) ->
+    Resources = rebar_state:resources(State),
+    #{repos := Repos} = rebar_resource:find_resource_state(pkg, Resources),
+    get_repo_config(RepoName, Repos).
+
+merge_with_base_and_auth(Repos, BaseConfig, Auth) ->
+    [maps:merge(maps:get(maps:get(name, Repo), Auth, #{}),
+                maps:merge(Repo, BaseConfig)) || Repo <- Repos].
+
+%% A user's list of repos are merged by name while keeping the order
+%% intact. The order is based on the first use of a repo by name in the
+%% list. The default repo is appended to the user's list.
+repos(HexConfig) ->
+    HexDefaultConfig = default_repo(),
+    case [R || R <- HexConfig, element(1, R) =:= repos] of
+        [] ->
+            [HexDefaultConfig];
+        %% we only care if the first element is a replace entry
+        [{repos, replace, Repos} | _]->
+            merge_repos(Repos);
+        Repos ->
+            RepoList = repo_list(Repos),
+            merge_repos(RepoList ++ [HexDefaultConfig])
+    end.
+
+-spec merge_repos([repo()]) -> [repo()].
+merge_repos(Repos) ->
+    lists:foldl(fun(R=#{name := Name}, ReposAcc) ->
+                        %% private organizations include the parent repo before a :
+                        case rebar_string:split(Name, <<":">>) of
+                            [Repo, Org] ->
+                                update_repo_list(R#{name => Name,
+                                                    organization => Org,
+                                                    parent => Repo}, ReposAcc);
+                            _ ->
+                                update_repo_list(R, ReposAcc)
+                        end
+                end, [], Repos).
+
+update_organizations(Repos) ->
+    lists:map(fun(Repo=#{parent := ParentName}) ->
+                      {ok, Parent} = get_repo_config(ParentName, Repos),
+                      maps:merge(Parent, Repo);
+                 (Repo) ->
+                      Repo
+              end, Repos).
+
+update_repo_list(R=#{name := N}, [H=#{name := HN} | Rest]) when N =:= HN ->
+    [maps:merge(R, H) | Rest];
+update_repo_list(R, [H | Rest]) ->
+    [H | update_repo_list(R, Rest)];
+update_repo_list(R, []) ->
+    [R].
+
+default_repo() ->
+    HexDefaultConfig = hex_core:default_config(),
+    HexDefaultConfig#{name => ?PUBLIC_HEX_REPO}.
+
+repo_list([]) ->
+    [];
+repo_list([{repos, Repos} | T]) ->
+    Repos ++ repo_list(T);
+repo_list([{repos, replace, Repos} | T]) ->
+    Repos ++ repo_list(T).
+
+%% auth functions
+
+%% authentication is in a separate config file because the hex plugin updates it
+
+-spec auth_config_file(rebar_state:t()) -> file:filename_all().
+auth_config_file(State) ->
+    filename:join(rebar_dir:global_config_dir(State), ?HEX_AUTH_FILE).
+
+-spec auth_config(rebar_state:t()) -> map().
+auth_config(State) ->
+    case file:consult(auth_config_file(State)) of
+        {ok, [Config]} ->
+            Config;
+        _ ->
+            #{}
+    end.
+
+-spec update_auth_config(map(), rebar_state:t()) -> ok.
+update_auth_config(Updates, State) ->
+    Config = auth_config(State),
+    NewConfig = iolist_to_binary([io_lib:print(maps:merge(Config, Updates)) | ".\n"]),
+    ok = file:write_file(auth_config_file(State), NewConfig).

src/rebar_pkg_resource.erl

@@ -16,7 +16,7 @@
 
 -ifdef(TEST).
 %% exported for test purposes
--export([store_etag_in_cache/2, repos/1, merge_repos/1]).
+-export([store_etag_in_cache/2]).
 -endif.
 
 -include("rebar.hrl").
@@ -30,13 +30,6 @@
 -type download_result() :: {bad_download, binary() | string()} |
                            {fetch_fail, _, _} | cached_result().
 
--type repo() :: #{name => unicode:unicode_binary(),
-                  api_url => binary(),
-                  api_key => binary(),
-                  repo_url => binary(),
-                  repo_public_key => binary(),
-                  repo_verify => binary()}.
-
 %%==============================================================================
 %% Public API
 %%==============================================================================
@@ -48,53 +41,10 @@ init(State) ->
                    http_user_agent_fragment =>
                        <<"(rebar3/", (list_to_binary(Vsn))/binary, ") (httpc)">>,
                    http_adapter_config => #{profile => rebar}},
-    HexConfig = rebar_state:get(State, hex, []),
-    Repos = repos(HexConfig),
-    %% add base config entries that are specific to use by rebar3 and not overridable
-    Repos1 = [maps:merge(Repo, BaseConfig) || Repo <- Repos],
-    {ok, #{repos => Repos1,
+    Repos = rebar_hex_repos:from_state(BaseConfig, State),
+    {ok, #{repos => Repos,
            base_config => BaseConfig}}.
 
-%% A user's list of repos are merged by name while keeping the order
-%% intact. The order is based on the first use of a repo by name in the
-%% list. The default repo is appended to the user's list.
-repos(HexConfig) ->
-    HexDefaultConfig = default_repo(),
-    case [R || R <- HexConfig, element(1, R) =:= repos] of
-        [] ->
-            [HexDefaultConfig];
-        %% we only care if the first element is a replace entry
-        [{repos, replace, Repos} | _]->
-            merge_repos(Repos);
-        Repos ->
-            RepoList = repo_list(Repos),
-            merge_repos(RepoList ++ [HexDefaultConfig])
-    end.
-
--spec merge_repos([repo()]) -> [repo()].
-merge_repos(Repos) ->
-    lists:foldl(fun(R, ReposAcc) ->
-                        update_repo_list(R, ReposAcc)
-                end, [], Repos).
-
-update_repo_list(R=#{name := N}, [H=#{name := HN} | Rest]) when N =:= HN ->
-    [maps:merge(R, H) | Rest];
-update_repo_list(R, [H | Rest]) ->
-    [H | update_repo_list(R, Rest)];
-update_repo_list(R, []) ->
-    [R].
-
-default_repo() ->
-    HexDefaultConfig = hex_core:default_config(),
-    HexDefaultConfig#{name => <<"hexpm">>}.
-
-repo_list([]) ->
-    [];
-repo_list([{repos, Repos} | T]) ->
-    Repos ++ repo_list(T);
-repo_list([{repos, replace, Repos} | T]) ->
-    Repos ++ repo_list(T).
-
 -spec lock(AppDir, Source) -> Res when
       AppDir :: file:name(),
       Source :: tuple(),

src/rebar_string.erl

@@ -1,7 +1,7 @@
 %%% @doc Compatibility module for string functionality
 %%% for pre- and post-unicode support.
 -module(rebar_string).
--export([join/2, lexemes/2, trim/3, uppercase/1, lowercase/1, chr/2]).
+-export([join/2, split/2, lexemes/2, trim/3, uppercase/1, lowercase/1, chr/2]).
 
 -ifdef(unicode_str).
 
@@ -15,6 +15,7 @@ join([], Sep) when is_list(Sep) ->
 join([H|T], Sep) ->
     H ++ lists:append([Sep ++ X || X <- T]).
 
+split(Str, SearchPattern) -> string:split(Str, SearchPattern).
 lexemes(Str, SepList) -> string:lexemes(Str, SepList).
 trim(Str, Direction, Cluster=[_]) -> string:trim(Str, Direction, Cluster).
 uppercase(Str) -> string:uppercase(Str).
@@ -27,6 +28,8 @@ chr([], _C, _I) -> 0.
 -else.
 
 join(Strings, Separator) -> string:join(Strings, Separator).
+split(Str, SearchPattern) when is_list(Str) -> string:split(Str, SearchPattern);
+split(Str, SearchPattern) when is_binary(Str) -> binary:split(Str, SearchPattern).
 lexemes(Str, SepList) -> string:tokens(Str, SepList).
 trim(Str, Direction, [Char]) ->
     Dir = case Direction of

test/rebar_pkg_repos_SUITE.erl

@@ -8,7 +8,8 @@
 -include("rebar.hrl").
 
 all() ->
-    [default_repo, repo_merging, repo_replacing, {group, resolve_version}].
+    [default_repo, repo_merging, repo_replacing,
+     auth_merging, organization_merging, {group, resolve_version}].
 
 groups() ->
     [{resolve_version, [use_first_repo_match, use_exact_with_hash, fail_repo_update,
@@ -101,9 +102,21 @@ init_per_testcase(ignore_match_in_excluded_repo, Config) ->
                 fun(_State) -> true end),
 
     [{state, State} | Config];
+init_per_testcase(auth_merging, Config) ->
+    meck:new(file, [passthrough, no_link, unstick]),
+    meck:new(rebar_packages, [passthrough, no_link]),
+    Config;
+init_per_testcase(organization_merging, Config) ->
+    meck:new(file, [passthrough, no_link, unstick]),
+    meck:new(rebar_packages, [passthrough, no_link]),
+    Config;
 init_per_testcase(_, Config) ->
     Config.
 
+end_per_testcase(Case, _Config) when Case =:= auth_merging ;
+                                     Case =:= organization_merging ->
+    meck:unload(file),
+    meck:unload(rebar_packages);
 end_per_testcase(Case, _Config) when Case =:= use_first_repo_match ;
                                      Case =:= use_exact_with_hash ;
                                      Case =:= fail_repo_update ;
@@ -117,7 +130,7 @@ default_repo(_Config) ->
     Repo1 = #{name => <<"hexpm">>,
               api_key => <<"asdf">>},
 
-    MergedRepos = rebar_pkg_resource:repos([{repos, [Repo1]}]),
+    MergedRepos = rebar_hex_repos:repos([{repos, [Repo1]}]),
 
     ?assertMatch([#{name := <<"hexpm">>,
                     api_key := <<"asdf">>,
@@ -130,7 +143,7 @@ repo_merging(_Config) ->
     Repo2 = #{name => <<"repo-2">>,
               repo_url => <<"repo-2/repo">>,
               repo_verify => false},
-    Result = rebar_pkg_resource:merge_repos([Repo1, Repo2,
+    Result = rebar_hex_repos:merge_repos([Repo1, Repo2,
                                              #{name => <<"repo-2">>,
                                                api_url => <<"repo-2/api">>,
                                                repo_url => <<"bad url">>,
@@ -139,7 +152,6 @@ repo_merging(_Config) ->
                                                api_url => <<"bad url">>,
                                                repo_verify => true},
                                              #{name => <<"repo-2">>,
-                                               organization => <<"repo-2-org">>,
                                                api_url => <<"repo-2/api-2">>,
                                                repo_url => <<"other/repo">>}]),
     ?assertMatch([#{name := <<"repo-1">>,
@@ -148,7 +160,6 @@ repo_merging(_Config) ->
                   #{name := <<"repo-2">>,
                     api_url := <<"repo-2/api">>,
                     repo_url := <<"repo-2/repo">>,
-                    organization := <<"repo-2-org">>,
                     repo_verify := false}], Result).
 
 repo_replacing(_Config) ->
@@ -159,19 +170,78 @@ repo_replacing(_Config) ->
               repo_verify => false},
 
     ?assertMatch([Repo1, Repo2, #{name := <<"hexpm">>}],
-                 rebar_pkg_resource:repos([{repos, [Repo1]},
+                 rebar_hex_repos:repos([{repos, [Repo1]},
                                            {repos, [Repo2]}])),
 
     %% use of replace is ignored if found in later entries than the first
     ?assertMatch([Repo1, Repo2, #{name := <<"hexpm">>}],
-                 rebar_pkg_resource:repos([{repos, [Repo1]},
+                 rebar_hex_repos:repos([{repos, [Repo1]},
                                            {repos, replace, [Repo2]}])),
 
     ?assertMatch([Repo1],
-                 rebar_pkg_resource:repos([{repos, replace, [Repo1]},
+                 rebar_hex_repos:repos([{repos, replace, [Repo1]},
                                            {repos, [Repo2]}])).
 
+auth_merging(_Config) ->
+    Repo1 = #{name => <<"repo-1">>,
+              api_url => <<"repo-1/api">>},
+    Repo2 = #{name => <<"repo-2">>,
+              repo_url => <<"repo-2/repo">>,
+              repo_verify => false},
+
+    State = rebar_state:new([{hex, [{repos, [Repo1, Repo2]}]}]),
+    meck:expect(file, consult,
+                fun(_) ->
+                        {ok, [#{<<"repo-1">> => #{read_key => <<"read key">>,
+                                                 write_key => <<"write key">>},
+                               <<"repo-2">> => #{read_key => <<"read key 2">>,
+                                                 repos_key => <<"repos key 2">>,
+                                                 write_key => <<"write key 2">>},
+                                <<"hexpm">> => #{write_key => <<"write key hexpm">>}}]}
+                end),
+
+    ?assertMatch({ok, #{repos := [#{name := <<"repo-1">>,
+                                    read_key := <<"read key">>,
+                                    write_key := <<"write key">>},
+                                  #{name := <<"repo-2">>,
+                                    read_key := <<"read key 2">>,
+                                    repos_key := <<"repos key 2">>,
+                                    write_key := <<"write key 2">>},
+                                  #{name := <<"hexpm">>,
+                                    write_key := <<"write key hexpm">>}]}}, rebar_pkg_resource:init(State)),
 
+    ok.
+
+organization_merging(_Config) ->
+    Repo1 = #{name => <<"hexpm:repo-1">>,
+              api_url => <<"repo-1/api">>},
+    Repo2 = #{name => <<"hexpm:repo-2">>,
+              repo_url => <<"repo-2/repo">>,
+              repo_verify => false},
+
+    State = rebar_state:new([{hex, [{repos, [Repo1, Repo2]}]}]),
+    meck:expect(file, consult,
+                fun(_) ->
+                        {ok, [#{<<"hexpm:repo-1">> => #{read_key => <<"read key">>},
+                                <<"hexpm:repo-2">> => #{read_key => <<"read key 2">>,
+                                                        repos_key => <<"repos key 2">>,
+                                                        write_key => <<"write key 2">>},
+                                <<"hexpm">> => #{write_key => <<"write key hexpm">>}}]}
+                end),
+
+    ?assertMatch({ok, #{repos := [#{name := <<"hexpm:repo-1">>,
+                                    parent := <<"hexpm">>,
+                                    read_key := <<"read key">>,
+                                    write_key := <<"write key hexpm">>},
+                                  #{name := <<"hexpm:repo-2">>,
+                                    parent := <<"hexpm">>,
+                                    read_key := <<"read key 2">>,
+                                    repos_key := <<"repos key 2">>,
+                                    write_key := <<"write key 2">>},
+                                  #{name := <<"hexpm">>,
+                                    write_key := <<"write key hexpm">>}]}}, rebar_pkg_resource:init(State)),
+
+    ok.
 
 use_first_repo_match(Config) ->
     State = ?config(state, Config),