From fbf74f04b2c97ca279926e58d747993055b5e17e Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Fri, 7 Sep 2018 17:29:14 -0600 Subject: [PATCH] add hex auth handling for repos (#1874) auth token are kept in a hex.config file that is modified by the rebar3 hex plugin. Repo names that have a : separating a parent and child are considered organizations. The parent repo's auth will be included with the child. So an organization named hexpm:rebar3_test will include any hexpm auth tokens found in the rebar3_test organization's configuration. --- src/rebar.hrl | 2 + src/rebar_hex_repos.erl | 122 +++++++++++++++++++++++++++++++++ src/rebar_pkg_resource.erl | 56 +-------------- src/rebar_string.erl | 5 +- test/rebar_pkg_repos_SUITE.erl | 86 ++++++++++++++++++++--- 5 files changed, 209 insertions(+), 62 deletions(-) create mode 100644 src/rebar_hex_repos.erl diff --git a/src/rebar.hrl b/src/rebar.hrl index e6c07280..4ea472a4 100644 --- a/src/rebar.hrl +++ b/src/rebar.hrl @@ -30,6 +30,8 @@ -define(PACKAGE_INDEX_VERSION, 4). -define(PACKAGE_TABLE, package_index_v4). -define(INDEX_FILE, "packages-v4.idx"). +-define(HEX_AUTH_FILE, "hex.config"). +-define(PUBLIC_HEX_REPO, <<"hexpm">>). %% the package record is used in a select match spec which upsets dialyzer %% this is the suggested workaround from Tobias diff --git a/src/rebar_hex_repos.erl b/src/rebar_hex_repos.erl new file mode 100644 index 00000000..60d1aa83 --- /dev/null +++ b/src/rebar_hex_repos.erl @@ -0,0 +1,122 @@ +-module(rebar_hex_repos). + +-export([from_state/2, + get_repo_config/2, + auth_config/1, + update_auth_config/2]). + +-ifdef(TEST). +%% exported for test purposes +-export([repos/1, merge_repos/1]). +-endif. + +-include("rebar.hrl"). + +-type repo() :: #{name => unicode:unicode_binary(), + api_url => binary(), + api_key => binary(), + repo_url => binary(), + repo_public_key => binary(), + repo_verify => binary()}. + +from_state(BaseConfig, State) -> + HexConfig = rebar_state:get(State, hex, []), + Repos = repos(HexConfig), + %% auth is stored in a separate config file since the plugin generates and modifies it + Auth = ?MODULE:auth_config(State), + %% add base config entries that are specific to use by rebar3 and not overridable + Repos1 = merge_with_base_and_auth(Repos, BaseConfig, Auth), + %% merge organizations parent repo options into each oraganization repo + update_organizations(Repos1). + +-spec get_repo_config(unicode:unicode_binary(), rebar_state:t() | [hex_core:config()]) + -> {ok, hex_core:config()} | error. +get_repo_config(RepoName, Repos) when is_list(Repos) -> + ec_lists:find(fun(#{name := N}) -> N =:= RepoName end, Repos); +get_repo_config(RepoName, State) -> + Resources = rebar_state:resources(State), + #{repos := Repos} = rebar_resource:find_resource_state(pkg, Resources), + get_repo_config(RepoName, Repos). + +merge_with_base_and_auth(Repos, BaseConfig, Auth) -> + [maps:merge(maps:get(maps:get(name, Repo), Auth, #{}), + maps:merge(Repo, BaseConfig)) || Repo <- Repos]. + +%% A user's list of repos are merged by name while keeping the order +%% intact. The order is based on the first use of a repo by name in the +%% list. The default repo is appended to the user's list. +repos(HexConfig) -> + HexDefaultConfig = default_repo(), + case [R || R <- HexConfig, element(1, R) =:= repos] of + [] -> + [HexDefaultConfig]; + %% we only care if the first element is a replace entry + [{repos, replace, Repos} | _]-> + merge_repos(Repos); + Repos -> + RepoList = repo_list(Repos), + merge_repos(RepoList ++ [HexDefaultConfig]) + end. + +-spec merge_repos([repo()]) -> [repo()]. +merge_repos(Repos) -> + lists:foldl(fun(R=#{name := Name}, ReposAcc) -> + %% private organizations include the parent repo before a : + case rebar_string:split(Name, <<":">>) of + [Repo, Org] -> + update_repo_list(R#{name => Name, + organization => Org, + parent => Repo}, ReposAcc); + _ -> + update_repo_list(R, ReposAcc) + end + end, [], Repos). + +update_organizations(Repos) -> + lists:map(fun(Repo=#{parent := ParentName}) -> + {ok, Parent} = get_repo_config(ParentName, Repos), + maps:merge(Parent, Repo); + (Repo) -> + Repo + end, Repos). + +update_repo_list(R=#{name := N}, [H=#{name := HN} | Rest]) when N =:= HN -> + [maps:merge(R, H) | Rest]; +update_repo_list(R, [H | Rest]) -> + [H | update_repo_list(R, Rest)]; +update_repo_list(R, []) -> + [R]. + +default_repo() -> + HexDefaultConfig = hex_core:default_config(), + HexDefaultConfig#{name => ?PUBLIC_HEX_REPO}. + +repo_list([]) -> + []; +repo_list([{repos, Repos} | T]) -> + Repos ++ repo_list(T); +repo_list([{repos, replace, Repos} | T]) -> + Repos ++ repo_list(T). + +%% auth functions + +%% authentication is in a separate config file because the hex plugin updates it + +-spec auth_config_file(rebar_state:t()) -> file:filename_all(). +auth_config_file(State) -> + filename:join(rebar_dir:global_config_dir(State), ?HEX_AUTH_FILE). + +-spec auth_config(rebar_state:t()) -> map(). +auth_config(State) -> + case file:consult(auth_config_file(State)) of + {ok, [Config]} -> + Config; + _ -> + #{} + end. + +-spec update_auth_config(map(), rebar_state:t()) -> ok. +update_auth_config(Updates, State) -> + Config = auth_config(State), + NewConfig = iolist_to_binary([io_lib:print(maps:merge(Config, Updates)) | ".\n"]), + ok = file:write_file(auth_config_file(State), NewConfig). diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl index b8f00177..4856b894 100644 --- a/src/rebar_pkg_resource.erl +++ b/src/rebar_pkg_resource.erl @@ -16,7 +16,7 @@ -ifdef(TEST). %% exported for test purposes --export([store_etag_in_cache/2, repos/1, merge_repos/1]). +-export([store_etag_in_cache/2]). -endif. -include("rebar.hrl"). @@ -30,13 +30,6 @@ -type download_result() :: {bad_download, binary() | string()} | {fetch_fail, _, _} | cached_result(). --type repo() :: #{name => unicode:unicode_binary(), - api_url => binary(), - api_key => binary(), - repo_url => binary(), - repo_public_key => binary(), - repo_verify => binary()}. - %%============================================================================== %% Public API %%============================================================================== @@ -48,53 +41,10 @@ init(State) -> http_user_agent_fragment => <<"(rebar3/", (list_to_binary(Vsn))/binary, ") (httpc)">>, http_adapter_config => #{profile => rebar}}, - HexConfig = rebar_state:get(State, hex, []), - Repos = repos(HexConfig), - %% add base config entries that are specific to use by rebar3 and not overridable - Repos1 = [maps:merge(Repo, BaseConfig) || Repo <- Repos], - {ok, #{repos => Repos1, + Repos = rebar_hex_repos:from_state(BaseConfig, State), + {ok, #{repos => Repos, base_config => BaseConfig}}. -%% A user's list of repos are merged by name while keeping the order -%% intact. The order is based on the first use of a repo by name in the -%% list. The default repo is appended to the user's list. -repos(HexConfig) -> - HexDefaultConfig = default_repo(), - case [R || R <- HexConfig, element(1, R) =:= repos] of - [] -> - [HexDefaultConfig]; - %% we only care if the first element is a replace entry - [{repos, replace, Repos} | _]-> - merge_repos(Repos); - Repos -> - RepoList = repo_list(Repos), - merge_repos(RepoList ++ [HexDefaultConfig]) - end. - --spec merge_repos([repo()]) -> [repo()]. -merge_repos(Repos) -> - lists:foldl(fun(R, ReposAcc) -> - update_repo_list(R, ReposAcc) - end, [], Repos). - -update_repo_list(R=#{name := N}, [H=#{name := HN} | Rest]) when N =:= HN -> - [maps:merge(R, H) | Rest]; -update_repo_list(R, [H | Rest]) -> - [H | update_repo_list(R, Rest)]; -update_repo_list(R, []) -> - [R]. - -default_repo() -> - HexDefaultConfig = hex_core:default_config(), - HexDefaultConfig#{name => <<"hexpm">>}. - -repo_list([]) -> - []; -repo_list([{repos, Repos} | T]) -> - Repos ++ repo_list(T); -repo_list([{repos, replace, Repos} | T]) -> - Repos ++ repo_list(T). - -spec lock(AppDir, Source) -> Res when AppDir :: file:name(), Source :: tuple(), diff --git a/src/rebar_string.erl b/src/rebar_string.erl index 47cb15cd..d03b14e9 100644 --- a/src/rebar_string.erl +++ b/src/rebar_string.erl @@ -1,7 +1,7 @@ %%% @doc Compatibility module for string functionality %%% for pre- and post-unicode support. -module(rebar_string). --export([join/2, lexemes/2, trim/3, uppercase/1, lowercase/1, chr/2]). +-export([join/2, split/2, lexemes/2, trim/3, uppercase/1, lowercase/1, chr/2]). -ifdef(unicode_str). @@ -15,6 +15,7 @@ join([], Sep) when is_list(Sep) -> join([H|T], Sep) -> H ++ lists:append([Sep ++ X || X <- T]). +split(Str, SearchPattern) -> string:split(Str, SearchPattern). lexemes(Str, SepList) -> string:lexemes(Str, SepList). trim(Str, Direction, Cluster=[_]) -> string:trim(Str, Direction, Cluster). uppercase(Str) -> string:uppercase(Str). @@ -27,6 +28,8 @@ chr([], _C, _I) -> 0. -else. join(Strings, Separator) -> string:join(Strings, Separator). +split(Str, SearchPattern) when is_list(Str) -> string:split(Str, SearchPattern); +split(Str, SearchPattern) when is_binary(Str) -> binary:split(Str, SearchPattern). lexemes(Str, SepList) -> string:tokens(Str, SepList). trim(Str, Direction, [Char]) -> Dir = case Direction of diff --git a/test/rebar_pkg_repos_SUITE.erl b/test/rebar_pkg_repos_SUITE.erl index 97e2e3c0..d7e1d06d 100644 --- a/test/rebar_pkg_repos_SUITE.erl +++ b/test/rebar_pkg_repos_SUITE.erl @@ -8,7 +8,8 @@ -include("rebar.hrl"). all() -> - [default_repo, repo_merging, repo_replacing, {group, resolve_version}]. + [default_repo, repo_merging, repo_replacing, + auth_merging, organization_merging, {group, resolve_version}]. groups() -> [{resolve_version, [use_first_repo_match, use_exact_with_hash, fail_repo_update, @@ -101,9 +102,21 @@ init_per_testcase(ignore_match_in_excluded_repo, Config) -> fun(_State) -> true end), [{state, State} | Config]; +init_per_testcase(auth_merging, Config) -> + meck:new(file, [passthrough, no_link, unstick]), + meck:new(rebar_packages, [passthrough, no_link]), + Config; +init_per_testcase(organization_merging, Config) -> + meck:new(file, [passthrough, no_link, unstick]), + meck:new(rebar_packages, [passthrough, no_link]), + Config; init_per_testcase(_, Config) -> Config. +end_per_testcase(Case, _Config) when Case =:= auth_merging ; + Case =:= organization_merging -> + meck:unload(file), + meck:unload(rebar_packages); end_per_testcase(Case, _Config) when Case =:= use_first_repo_match ; Case =:= use_exact_with_hash ; Case =:= fail_repo_update ; @@ -117,7 +130,7 @@ default_repo(_Config) -> Repo1 = #{name => <<"hexpm">>, api_key => <<"asdf">>}, - MergedRepos = rebar_pkg_resource:repos([{repos, [Repo1]}]), + MergedRepos = rebar_hex_repos:repos([{repos, [Repo1]}]), ?assertMatch([#{name := <<"hexpm">>, api_key := <<"asdf">>, @@ -130,7 +143,7 @@ repo_merging(_Config) -> Repo2 = #{name => <<"repo-2">>, repo_url => <<"repo-2/repo">>, repo_verify => false}, - Result = rebar_pkg_resource:merge_repos([Repo1, Repo2, + Result = rebar_hex_repos:merge_repos([Repo1, Repo2, #{name => <<"repo-2">>, api_url => <<"repo-2/api">>, repo_url => <<"bad url">>, @@ -139,7 +152,6 @@ repo_merging(_Config) -> api_url => <<"bad url">>, repo_verify => true}, #{name => <<"repo-2">>, - organization => <<"repo-2-org">>, api_url => <<"repo-2/api-2">>, repo_url => <<"other/repo">>}]), ?assertMatch([#{name := <<"repo-1">>, @@ -148,7 +160,6 @@ repo_merging(_Config) -> #{name := <<"repo-2">>, api_url := <<"repo-2/api">>, repo_url := <<"repo-2/repo">>, - organization := <<"repo-2-org">>, repo_verify := false}], Result). repo_replacing(_Config) -> @@ -159,19 +170,78 @@ repo_replacing(_Config) -> repo_verify => false}, ?assertMatch([Repo1, Repo2, #{name := <<"hexpm">>}], - rebar_pkg_resource:repos([{repos, [Repo1]}, + rebar_hex_repos:repos([{repos, [Repo1]}, {repos, [Repo2]}])), %% use of replace is ignored if found in later entries than the first ?assertMatch([Repo1, Repo2, #{name := <<"hexpm">>}], - rebar_pkg_resource:repos([{repos, [Repo1]}, + rebar_hex_repos:repos([{repos, [Repo1]}, {repos, replace, [Repo2]}])), ?assertMatch([Repo1], - rebar_pkg_resource:repos([{repos, replace, [Repo1]}, + rebar_hex_repos:repos([{repos, replace, [Repo1]}, {repos, [Repo2]}])). +auth_merging(_Config) -> + Repo1 = #{name => <<"repo-1">>, + api_url => <<"repo-1/api">>}, + Repo2 = #{name => <<"repo-2">>, + repo_url => <<"repo-2/repo">>, + repo_verify => false}, + + State = rebar_state:new([{hex, [{repos, [Repo1, Repo2]}]}]), + meck:expect(file, consult, + fun(_) -> + {ok, [#{<<"repo-1">> => #{read_key => <<"read key">>, + write_key => <<"write key">>}, + <<"repo-2">> => #{read_key => <<"read key 2">>, + repos_key => <<"repos key 2">>, + write_key => <<"write key 2">>}, + <<"hexpm">> => #{write_key => <<"write key hexpm">>}}]} + end), + + ?assertMatch({ok, #{repos := [#{name := <<"repo-1">>, + read_key := <<"read key">>, + write_key := <<"write key">>}, + #{name := <<"repo-2">>, + read_key := <<"read key 2">>, + repos_key := <<"repos key 2">>, + write_key := <<"write key 2">>}, + #{name := <<"hexpm">>, + write_key := <<"write key hexpm">>}]}}, rebar_pkg_resource:init(State)), + ok. + +organization_merging(_Config) -> + Repo1 = #{name => <<"hexpm:repo-1">>, + api_url => <<"repo-1/api">>}, + Repo2 = #{name => <<"hexpm:repo-2">>, + repo_url => <<"repo-2/repo">>, + repo_verify => false}, + + State = rebar_state:new([{hex, [{repos, [Repo1, Repo2]}]}]), + meck:expect(file, consult, + fun(_) -> + {ok, [#{<<"hexpm:repo-1">> => #{read_key => <<"read key">>}, + <<"hexpm:repo-2">> => #{read_key => <<"read key 2">>, + repos_key => <<"repos key 2">>, + write_key => <<"write key 2">>}, + <<"hexpm">> => #{write_key => <<"write key hexpm">>}}]} + end), + + ?assertMatch({ok, #{repos := [#{name := <<"hexpm:repo-1">>, + parent := <<"hexpm">>, + read_key := <<"read key">>, + write_key := <<"write key hexpm">>}, + #{name := <<"hexpm:repo-2">>, + parent := <<"hexpm">>, + read_key := <<"read key 2">>, + repos_key := <<"repos key 2">>, + write_key := <<"write key 2">>}, + #{name := <<"hexpm">>, + write_key := <<"write key hexpm">>}]}}, rebar_pkg_resource:init(State)), + + ok. use_first_repo_match(Config) -> State = ?config(state, Config),