From d7e0f0946143d6f921451cb0735f952c0f362eca Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Tue, 4 Sep 2018 17:10:22 -0600 Subject: [PATCH] wip: support list of repos for hex packages (#1866) * support list of repos for hex packages repos are defined under the hex key in rebar configs. They can be defined at the top level of a project or globally, but not in profiles and the repos configured in dependencies are also ignored. Searching for packages involves first checking for a match in the local repo index cache, in the order repos are defined. If not found each repo is checked through the hex api for any known versions of the package and the first repo with a version that fits the constraint is used. * add {repos, replace, []} for overriding the global & default repos --- bootstrap | 3 +- rebar.config | 3 +- rebar.lock | 6 +- src/rebar.app.src | 1 + src/rebar.hrl | 9 +- src/rebar3.erl | 19 +- src/rebar_app_utils.erl | 42 ++-- src/rebar_git_resource.erl | 7 +- src/rebar_hg_resource.erl | 7 +- src/rebar_packages.erl | 313 ++++++++++++++++++++---------- src/rebar_pkg_resource.erl | 145 +++++++++----- src/rebar_prv_install_deps.erl | 2 +- src/rebar_prv_packages.erl | 71 ++++--- src/rebar_prv_repos.erl | 47 +++++ src/rebar_prv_update.erl | 14 +- src/rebar_prv_upgrade.erl | 12 +- src/rebar_resource.erl | 2 +- src/rebar_state.erl | 37 ++-- src/rebar_utils.erl | 15 +- test/mock_git_resource.erl | 2 +- test/mock_pkg_resource.erl | 42 ++-- test/rebar_deps_SUITE.erl | 16 +- test/rebar_install_deps_SUITE.erl | 2 +- test/rebar_localfs_resource.erl | 7 +- test/rebar_pkg_SUITE.erl | 81 +++----- test/rebar_pkg_alias_SUITE.erl | 18 +- test/rebar_pkg_repos_SUITE.erl | 257 ++++++++++++++++++++++++ test/rebar_test_utils.erl | 9 +- 28 files changed, 845 insertions(+), 344 deletions(-) create mode 100644 src/rebar_prv_repos.erl create mode 100644 test/rebar_pkg_repos_SUITE.erl diff --git a/bootstrap b/bootstrap index dfce9645..4196ed53 100755 --- a/bootstrap +++ b/bootstrap @@ -18,7 +18,8 @@ main(_) -> ,{erlware_commons, ["ec_dictionary.erl", "ec_vsn.erl"]} ,{parse_trans, ["parse_trans.erl", "parse_trans_pp.erl", "parse_trans_codegen.erl"]} - ,{certifi, []}], + ,{certifi, []} + ,{hex_core, []}], Deps = get_deps(), [fetch_and_compile(Dep, Deps) || Dep <- BaseDeps], diff --git a/rebar.config b/rebar.config index 861571dd..2d419275 100644 --- a/rebar.config +++ b/rebar.config @@ -11,8 +11,7 @@ {relx, "3.26.0"}, {cf, "0.2.2"}, {cth_readable, "1.4.2"}, - %% {hex_core, "0.1.1"}, - {hex_core, {git, "https://github.com/hexpm/hex_core.git", {branch, "master"}}}, + {hex_core, "0.2.0"}, {eunit_formatters, "0.5.0"}]}. {post_hooks, [{"(linux|darwin|solaris|freebsd|netbsd|openbsd)", diff --git a/rebar.lock b/rebar.lock index b479c772..152bfb78 100644 --- a/rebar.lock +++ b/rebar.lock @@ -6,10 +6,7 @@ {<<"erlware_commons">>,{pkg,<<"erlware_commons">>,<<"1.2.0">>},0}, {<<"eunit_formatters">>,{pkg,<<"eunit_formatters">>,<<"0.5.0">>},0}, {<<"getopt">>,{pkg,<<"getopt">>,<<"1.0.1">>},0}, - {<<"hex_core">>, - {git,"https://github.com/hexpm/hex_core.git", - {ref,"9f32aaf6c3b74c310da6f1e1ae1abe3f699a5b94"}}, - 0}, + {<<"hex_core">>,{pkg,<<"hex_core">>,<<"0.2.0">>},0}, {<<"parse_trans">>,{pkg,<<"parse_trans">>,<<"3.3.0">>},0}, {<<"providers">>,{pkg,<<"providers">>,<<"1.7.0">>},0}, {<<"relx">>,{pkg,<<"relx">>,<<"3.26.0">>},0}, @@ -23,6 +20,7 @@ {<<"erlware_commons">>, <<"2BAB99CF88941145767A502F1209886F1F0D31695EEF21978A30F15E645721E0">>}, {<<"eunit_formatters">>, <<"6A9133943D36A465D804C1C5B6E6839030434B8879C5600D7DDB5B3BAD4CCB59">>}, {<<"getopt">>, <<"C73A9FA687B217F2FF79F68A3B637711BB1936E712B521D8CE466B29CBF7808A">>}, + {<<"hex_core">>, <<"3A7EACCFB8ADD3FF05D950C10ED5BDB5D0C48C988EBBC5D7AE2A55498F0EFF1B">>}, {<<"parse_trans">>, <<"09765507A3C7590A784615CFD421D101AEC25098D50B89D7AA1D66646BC571C1">>}, {<<"providers">>, <<"BBF730563914328EC2511D205E6477A94831DB7297DE313B3872A2B26C562EAB">>}, {<<"relx">>, <<"DD645ECAA1AB1647DB80D3E9BCAE0B39ED0A536EF37245F6A74B114C6D0F4E87">>}, diff --git a/src/rebar.app.src b/src/rebar.app.src index 8e5eefab..530a79e4 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -68,6 +68,7 @@ rebar_prv_release, rebar_prv_relup, rebar_prv_report, + rebar_prv_repos, rebar_prv_shell, rebar_prv_state, rebar_prv_tar, diff --git a/src/rebar.hrl b/src/rebar.hrl index 63bda42f..e6c07280 100644 --- a/src/rebar.hrl +++ b/src/rebar.hrl @@ -36,10 +36,13 @@ %% http://erlang.org/pipermail/erlang-questions/2009-February/041445.html -type ms_field() :: '$1' | '_'. --record(package, {key :: {unicode:unicode_binary() | ms_field(), unicode:unicode_binary() | ms_field()}, +%% TODO: change package and requirement keys to be required (:=) after dropping support for OTP-18 +-record(package, {key :: {unicode:unicode_binary() | ms_field(), unicode:unicode_binary() | ms_field(), + unicode:unicode_binary() | ms_field()}, checksum :: binary() | ms_field(), - dependencies :: [#{package := unicode:unicode_binary(), - requirement := unicode:unicode_binary()}] | ms_field()}). + retired :: boolean() | ms_field(), + dependencies :: [#{package => unicode:unicode_binary(), + requirement => unicode:unicode_binary()}] | ms_field()}). -ifdef(namespaced_types). -type rebar_dict() :: dict:dict(). diff --git a/src/rebar3.erl b/src/rebar3.erl index ec8e9534..e87cb196 100644 --- a/src/rebar3.erl +++ b/src/rebar3.erl @@ -103,7 +103,7 @@ run(RawArgs) -> case erlang:system_info(version) of "6.1" -> ?WARN("Due to a filelib bug in Erlang 17.1 it is recommended" - "you update to a newer release.", []); + "you update to a newer release.", []); _ -> ok end, @@ -139,8 +139,14 @@ run_aux(State, RawArgs) -> rebar_state:set(State1, rebar_packages_cdn, CDN) end, + %% TODO: this means use of REBAR_PROFILE=profile will replace the repos with + %% the repos defined in the profile. But it will not work with `as profile`. + %% Maybe it shouldn't work with either to be consistent? + Resources = application:get_env(rebar, resources, []), + State2_ = rebar_state:create_resources(Resources, State2), + %% bootstrap test profile - State3 = rebar_state:add_to_profile(State2, test, test_state(State1)), + State3 = rebar_state:add_to_profile(State2_, test, test_state(State1)), %% Process each command, resetting any state between each one BaseDir = rebar_state:get(State, base_dir, ?DEFAULT_BASE_DIR), @@ -375,7 +381,11 @@ state_from_global_config(Config, GlobalConfigFile) -> %% We don't want to worry about global plugin install state effecting later %% usage. So we throw away the global profile state used for plugin install. - GlobalConfigThrowAway = rebar_state:current_profiles(GlobalConfig, [global]), + GlobalConfigThrowAway0 = rebar_state:current_profiles(GlobalConfig, [global]), + + Resources = application:get_env(rebar, resources, []), + GlobalConfigThrowAway = rebar_state:create_resources(Resources, GlobalConfigThrowAway0), + GlobalState = case rebar_state:get(GlobalConfigThrowAway, plugins, []) of [] -> GlobalConfigThrowAway; @@ -386,7 +396,8 @@ state_from_global_config(Config, GlobalConfigFile) -> end, GlobalPlugins = rebar_state:providers(GlobalState), GlobalConfig2 = rebar_state:set(GlobalConfig, plugins, []), - GlobalConfig3 = rebar_state:set(GlobalConfig2, {plugins, global}, rebar_state:get(GlobalConfigThrowAway, plugins, [])), + GlobalConfig3 = rebar_state:set(GlobalConfig2, {plugins, global}, + rebar_state:get(GlobalConfigThrowAway, plugins, [])), rebar_state:providers(rebar_state:new(GlobalConfig3, Config), GlobalPlugins). -spec test_state(rebar_state:t()) -> [{'extra_src_dirs',[string()]} | {'erl_opts',[any()]}]. diff --git a/src/rebar_app_utils.erl b/src/rebar_app_utils.erl index c1b6b219..2190a90a 100644 --- a/src/rebar_app_utils.erl +++ b/src/rebar_app_utils.erl @@ -252,34 +252,34 @@ expand_deps_sources(Dep, State) -> rebar_app_info:t() when Source :: rebar_resource:source(). update_source(AppInfo, {pkg, PkgName, PkgVsn, Hash}, State) -> - {ok, PkgVsn1} = rebar_packages:resolve_version(PkgName, PkgVsn, - ?PACKAGE_TABLE, State), - %% store the expected hash for the dependency - Hash1 = case Hash of - undefined -> % unknown, define the hash since we know the dep - fetch_checksum(PkgName, PkgVsn1, State); - _ -> % keep as is - Hash - end, - AppInfo1 = rebar_app_info:source(AppInfo, {pkg, PkgName, PkgVsn1, Hash1}), - Deps = rebar_packages:get_package_deps(PkgName - ,PkgVsn1 - ,State), - AppInfo2 = rebar_app_info:resource_type(rebar_app_info:deps(AppInfo1, Deps), pkg), - rebar_app_info:original_vsn(AppInfo2, PkgVsn1); + case rebar_packages:resolve_version(PkgName, PkgVsn, Hash, + ?PACKAGE_TABLE, State) of + {ok, Package, RepoConfig} -> + #package{key = {_, PkgVsn1, _}, + checksum = Hash1, + dependencies = Deps} = Package, + AppInfo1 = rebar_app_info:source(AppInfo, {pkg, PkgName, PkgVsn1, Hash1, RepoConfig}), + AppInfo2 = rebar_app_info:resource_type(rebar_app_info:deps(AppInfo1, Deps), pkg), + rebar_app_info:original_vsn(AppInfo2, PkgVsn1); + not_found -> + throw(?PRV_ERROR({missing_package, PkgName, PkgVsn})); + {error, {invalid_vsn, InvalidVsn}} -> + throw(?PRV_ERROR({invalid_vsn, PkgName, InvalidVsn})) + end; update_source(AppInfo, Source, _State) -> rebar_app_info:source(AppInfo, Source). -%% @doc grab the checksum for a given package --spec fetch_checksum(binary(), binary(), rebar_state:t()) - -> iodata() | no_return(). -fetch_checksum(PkgName, PkgVsn, State) -> - rebar_packages:registry_checksum(PkgName, PkgVsn, State). - %% @doc convert a given exception's payload into an io description. -spec format_error(any()) -> iolist(). +format_error({missing_package, Name, undefined}) -> + io_lib:format("Package not found in any repo: ~ts.", [rebar_utils:to_binary(Name)]); +format_error({missing_package, Name, Vsn}) -> + io_lib:format("Package not found in any repo: ~ts-~ts.", [rebar_utils:to_binary(Name), + rebar_utils:to_binary(Vsn)]); format_error({parse_dep, Dep}) -> io_lib:format("Failed parsing dep ~p", [Dep]); +format_error({invalid_vsn, Dep, InvalidVsn}) -> + io_lib:format("Dep ~ts has invalid version ~ts", [Dep, InvalidVsn]); format_error(Error) -> io_lib:format("~p", [Error]). diff --git a/src/rebar_git_resource.erl b/src/rebar_git_resource.erl index 3aa875f0..c695ea57 100644 --- a/src/rebar_git_resource.erl +++ b/src/rebar_git_resource.erl @@ -4,7 +4,8 @@ -behaviour(rebar_resource). --export([lock/2 +-export([init/1 + ,lock/2 ,download/3 ,needs_update/2 ,make_vsn/1]). @@ -14,6 +15,10 @@ %% Regex used for parsing scp style remote url -define(SCP_PATTERN, "\\A(?[^@]+)@(?[^:]+):(?.+)\\z"). +-spec init(rebar_state:t()) -> {ok, term()}. +init(_State) -> + {ok, #{}}. + lock(AppDir, {git, Url, _}) -> lock(AppDir, {git, Url}); lock(AppDir, {git, Url}) -> diff --git a/src/rebar_hg_resource.erl b/src/rebar_hg_resource.erl index abcca88c..21d4a809 100644 --- a/src/rebar_hg_resource.erl +++ b/src/rebar_hg_resource.erl @@ -4,13 +4,18 @@ -behaviour(rebar_resource). --export([lock/2 +-export([init/1 + ,lock/2 ,download/3 ,needs_update/2 ,make_vsn/1]). -include("rebar.hrl"). +-spec init(rebar_state:t()) -> {ok, term()}. +init(_State) -> + {ok, #{}}. + lock(AppDir, {hg, Url, _}) -> lock(AppDir, {hg, Url}); lock(AppDir, {hg, Url}) -> diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl index 0dfce1c1..d00222df 100644 --- a/src/rebar_packages.erl +++ b/src/rebar_packages.erl @@ -2,20 +2,19 @@ -export([get/2 ,get_all_names/1 - ,get_package_versions/3 - ,get_package_deps/3 + ,get_package_versions/4 + ,get_package_deps/4 ,new_package_table/0 ,load_and_verify_version/1 ,registry_dir/1 ,package_dir/1 - ,registry_checksum/3 - ,find_highest_matching/6 - ,find_highest_matching/4 - ,find_highest_matching_/6 + ,registry_checksum/4 + ,find_highest_matching/5 + ,find_highest_matching_/5 ,verify_table/1 ,format_error/1 - ,update_package/2 - ,resolve_version/4]). + ,update_package/3 + ,resolve_version/5]). -ifdef(TEST). -export([cmp_/4, cmpl_/4, valid_vsn/1]). @@ -31,55 +30,93 @@ -type package() :: pkg_name() | {pkg_name(), vsn()}. format_error({missing_package, Name, Vsn}) -> - io_lib:format("Package not found in registry: ~ts-~ts.", [rebar_utils:to_binary(Name), + io_lib:format("Package not found in any repo: ~ts-~ts.", [rebar_utils:to_binary(Name), rebar_utils:to_binary(Vsn)]); format_error({missing_package, Pkg}) -> - io_lib:format("Package not found in registry: ~p.", [Pkg]). + io_lib:format("Package not found in any repo: ~p.", [Pkg]). -spec get(hex_core:config(), binary()) -> {ok, map()} | {error, term()}. get(Config, Name) -> - case hex_api_package:get(Config, Name) of + try hex_api_package:get(Config, Name) of {ok, {200, _Headers, PkgInfo}} -> {ok, PkgInfo}; - _ -> - {error, blewup} + {ok, {404, _, _}} -> + {error, not_found}; + Error -> + ?DEBUG("Hex api request failed: ~p", [Error]), + {error, unknown} + catch + error:{badmatch, {error, {failed_connect, _}}} -> + {error, failed_to_connect}; + _:Exception -> + ?DEBUG("hex_api_package:get failed: ~p", [Exception]), + {error, unknown} end. + -spec get_all_names(rebar_state:t()) -> [binary()]. get_all_names(State) -> verify_table(State), - lists:usort(ets:select(?PACKAGE_TABLE, [{#package{key={'$1', '_'}, + lists:usort(ets:select(?PACKAGE_TABLE, [{#package{key={'$1', '_', '_'}, _='_'}, [], ['$1']}])). --spec get_package_versions(binary(), ets:tid(), rebar_state:t()) -> [vsn()]. -get_package_versions(Dep, Table, State) -> +-spec get_package_versions(unicode:unicode_binary(), unicode:unicode_binary(), ets:tid(), rebar_state:t()) + -> [vsn()]. +get_package_versions(Dep, Repo, Table, State) -> ?MODULE:verify_table(State), - ets:select(Table, [{#package{key={Dep,'$1'}, + ets:select(Table, [{#package{key={Dep,'$1', Repo}, _='_'}, [], ['$1']}]). + +get_package(Dep, Vsn, Hash, Repo, Table, State) -> + get_package(Dep, Vsn, Hash, false, [Repo], Table, State). + +-spec get_package(unicode:unicode_binary(), unicode:unicode_binary(), + binary() | undefined | '_', boolean() | '_', + unicode:unicode_binary() | '_' | list(), ets:tab(), rebar_state:t()) + -> {ok, #package{}} | not_found. +get_package(Dep, Vsn, undefined, Retired, Repo, Table, State) -> + get_package(Dep, Vsn, '_', Retired, Repo, Table, State); +get_package(Dep, Vsn, Hash, Retired, Repos, Table, State) when is_list(Repos) -> + ?MODULE:verify_table(State), + case ets:select(Table, [{#package{key={Dep, Vsn, Repo}, + checksum=Hash, + retired=Retired, + _='_'}, [], ['$_']} || Repo <- Repos]) of + %% have to allow multiple matches in the list for cases that Repo is `_` + [Package | _] -> + {ok, Package}; + _ -> + not_found + end; +get_package(Dep, Vsn, Hash, Retired, Repo, Table, State) -> + get_package(Dep, Vsn, Hash, Retired, [Repo], Table, State). + new_package_table() -> - ets:new(?PACKAGE_TABLE, [named_table, public, ordered_set, {keypos, 2}]), - ets:insert(package_index, {?PACKAGE_INDEX_VERSION, package_index_version}). + ?PACKAGE_TABLE = ets:new(?PACKAGE_TABLE, [named_table, public, ordered_set, {keypos, 2}]), + ets:insert(?PACKAGE_TABLE, {?PACKAGE_INDEX_VERSION, package_index_version}). --spec get_package_deps(binary(), vsn(), rebar_state:t()) -> [map()]. -get_package_deps(Name, Vsn, State) -> - try_lookup(?PACKAGE_TABLE, {Name, Vsn}, #package.dependencies, State). +-spec get_package_deps(unicode:unicode_binary(), unicode:unicode_binary(), vsn(), rebar_state:t()) + -> [map()]. +get_package_deps(Name, Vsn, Repo, State) -> + try_lookup(?PACKAGE_TABLE, {Name, Vsn, Repo}, #package.dependencies, State). --spec registry_checksum(binary(), vsn(), rebar_state:t()) -> binary(). -registry_checksum(Name, Vsn, State) -> - try_lookup(?PACKAGE_TABLE, {Name, Vsn}, #package.checksum, State). +-spec registry_checksum(unicode:unicode_binary(), vsn(), unicode:unicode_binary(), rebar_state:t()) + -> binary(). +registry_checksum(Name, Vsn, Repo, State) -> + try_lookup(?PACKAGE_TABLE, {Name, Vsn, Repo}, #package.checksum, State). -try_lookup(Table, Key, Element, State) -> +try_lookup(Table, Key={_, _, Repo}, Element, State) -> ?MODULE:verify_table(State), try ets:lookup_element(Table, Key, Element) catch _:_ -> - handle_missing_package(Key, State, fun(_) -> - ets:lookup_element(Table, Key, Element) - end) + handle_missing_package(Key, Repo, State, fun(_) -> + ets:lookup_element(Table, Key, Element) + end) end. load_and_verify_version(State) -> @@ -101,10 +138,10 @@ load_and_verify_version(State) -> new_package_table() end. -handle_missing_package(PkgKey, State, Fun) -> +handle_missing_package(PkgKey, Repo, State, Fun) -> Name = case PkgKey of - {N, Vsn} -> + {N, Vsn, _Repo} -> ?DEBUG("Package ~ts-~ts not found. Fetching registry updates for " "package and trying again...", [N, Vsn]), N; @@ -114,7 +151,7 @@ handle_missing_package(PkgKey, State, Fun) -> PkgKey end, - update_package(Name, State), + update_package(Name, Repo, State), try Fun(State) catch @@ -174,30 +211,27 @@ package_dir(State) -> %% `~> 2.1.3-dev` | `>= 2.1.3-dev and < 2.2.0` %% `~> 2.0` | `>= 2.0.0 and < 3.0.0` %% `~> 2.1` | `>= 2.1.0 and < 3.0.0` -find_highest_matching(Dep, Constraint, Table, State) -> - find_highest_matching(undefined, undefined, Dep, Constraint, Table, State). - -find_highest_matching(Pkg, PkgVsn, Dep, Constraint, Table, State) -> - try find_highest_matching_(Pkg, PkgVsn, Dep, Constraint, Table, State) of +find_highest_matching(Dep, Constraint, Repo, Table, State) -> + try find_highest_matching_(Dep, Constraint, Repo, Table, State) of none -> - handle_missing_package(Dep, State, + handle_missing_package(Dep, Repo, State, fun(State1) -> - find_highest_matching_(Pkg, PkgVsn, Dep, Constraint, Table, State1) + find_highest_matching_(Dep, Constraint, Repo, Table, State1) end); Result -> Result catch _:_ -> - handle_missing_package(Dep, State, + handle_missing_package(Dep, Repo, State, fun(State1) -> - find_highest_matching_(Pkg, PkgVsn, Dep, Constraint, Table, State1) + find_highest_matching_(Dep, Constraint, Repo, Table, State1) end) end. -find_highest_matching_(Pkg, PkgVsn, Dep, Constraint, Table, State) -> - try get_package_versions(Dep, Table, State) of +find_highest_matching_(Dep, Constraint, #{name := Repo}, Table, State) -> + try get_package_versions(Dep, Repo, Table, State) of [Vsn] -> - handle_single_vsn(Pkg, PkgVsn, Dep, Vsn, Constraint); + handle_single_vsn(Vsn, Constraint); Vsns -> case handle_vsns(Constraint, Vsns) of none -> @@ -221,20 +255,12 @@ handle_vsns(Constraint, Vsns) -> end end, none, Vsns). -handle_single_vsn(Pkg, PkgVsn, Dep, Vsn, Constraint) -> +handle_single_vsn(Vsn, Constraint) -> case ec_semver:pes(Vsn, Constraint) of true -> {ok, Vsn}; false -> - case {Pkg, PkgVsn} of - {undefined, undefined} -> - ?DEBUG("Only existing version of ~ts is ~ts which does not match constraint ~~> ~ts. " - "Using anyway, but it is not guaranteed to work.", [Dep, Vsn, Constraint]); - _ -> - ?DEBUG("[~ts:~ts] Only existing version of ~ts is ~ts which does not match constraint ~~> ~ts. " - "Using anyway, but it is not guaranteed to work.", [Pkg, PkgVsn, Dep, Vsn, Constraint]) - end, - {ok, Vsn} + none end. verify_table(State) -> @@ -252,47 +278,129 @@ parse_checksum(<>) -> parse_checksum(Checksum) -> Checksum. -update_package(Name, State) -> - Resources = rebar_state:resources(State), - #{hex_config := HexConfig} = rebar_resource:find_resource_state(pkg, Resources), - case hex_repo:get_package(HexConfig, Name) of +update_package(Name, RepoConfig=#{name := Repo}, State) -> + ?MODULE:verify_table(State), + try hex_repo:get_package(RepoConfig, Name) of {ok, {200, _Headers, #{releases := Releases}}} -> - _ = insert_releases(Name, Releases, ?PACKAGE_TABLE), + _ = insert_releases(Name, Releases, Repo, ?PACKAGE_TABLE), {ok, RegistryDir} = rebar_packages:registry_dir(State), PackageIndex = filename:join(RegistryDir, ?INDEX_FILE), ok = ets:tab2file(?PACKAGE_TABLE, PackageIndex); - _ -> + Error -> + ?DEBUG("Hex get_package request failed: ~p", [Error]), + %% TODO: add better log message. hex_core should export a format_error + ?WARN("Failed to update package from repo ~ts", [Repo]), + fail + catch + _:Exception -> + ?DEBUG("hex_repo:get_package failed for package ~p: ~p", [Name, Exception]), fail end. -insert_releases(Name, Releases, Table) -> +insert_releases(Name, Releases, Repo, Table) -> [true = ets:insert(Table, - #package{key={Name, Version}, + #package{key={Name, Version, Repo}, checksum=parse_checksum(Checksum), + retired=maps:get(retired, Release, false), dependencies=parse_deps(Dependencies)}) - || #{checksum := Checksum, - version := Version, - dependencies := Dependencies} <- Releases]. - -resolve_version(Dep, undefined, HexRegistry, State) -> - find_highest_matching(Dep, "0", HexRegistry, State); -resolve_version(Dep, DepVsn, HexRegistry, State) -> - case {valid_vsn(DepVsn), DepVsn} of - {false, Vsn} -> - {error, {invalid_vsn, Vsn}}; - {_, <<"~>", Vsn/binary>>} -> - highest_matching(Dep, rm_ws(Vsn), HexRegistry, State); - {_, <<">=", Vsn/binary>>} -> - cmp(Dep, rm_ws(Vsn), HexRegistry, State, fun ec_semver:gte/2); - {_, <<">", Vsn/binary>>} -> - cmp(Dep, rm_ws(Vsn), HexRegistry, State, fun ec_semver:gt/2); - {_, <<"<=", Vsn/binary>>} -> - cmpl(Dep, rm_ws(Vsn), HexRegistry, State, fun ec_semver:lte/2); - {_, <<"<", Vsn/binary>>} -> - cmpl(Dep, rm_ws(Vsn), HexRegistry, State, fun ec_semver:lt/2); - {_, <<"==", Vsn/binary>>} -> + || Release=#{checksum := Checksum, + version := Version, + dependencies := Dependencies} <- Releases]. + +-spec resolve_version(unicode:unicode_binary(), unicode:unicode_binary() | undefined, + binary() | undefined, + ets:tab(), rebar_state:t()) + -> {error, {invalid_vsn, unicode:unicode_binary()}} | + not_found | + {ok, #package{}, map()}. +%% if checksum is defined search for any matching repo matching pkg-vsn and checksum +resolve_version(Dep, DepVsn, Hash, HexRegistry, State) when is_binary(Hash) -> + Resources = rebar_state:resources(State), + #{repos := RepoConfigs} = rebar_resource:find_resource_state(pkg, Resources), + RepoNames = [RepoName || #{name := RepoName} <- RepoConfigs], + + %% allow retired packages when we have a checksum + case get_package(Dep, DepVsn, Hash, '_', RepoNames, HexRegistry, State) of + {ok, Package=#package{key={_, _, RepoName}}} -> + {ok, RepoConfig} = ec_lists:find(fun(#{name := N}) when N =:= RepoName -> + true; + (_) -> + false + end, RepoConfigs), + {ok, Package, RepoConfig}; + _ -> + Fun = fun(Repo) -> + case resolve_version_(Dep, DepVsn, Repo, HexRegistry, State) of + none -> + not_found; + {ok, Vsn} -> + get_package(Dep, Vsn, Hash, Repo, HexRegistry, State) + end + end, + handle_missing_no_exception(Fun, Dep, State) + end; +resolve_version(Dep, undefined, Hash, HexRegistry, State) -> + Fun = fun(Repo) -> + case highest_matching(Dep, "0", Repo, HexRegistry, State) of + none -> + not_found; + {ok, Vsn} -> + get_package(Dep, Vsn, Hash, Repo, HexRegistry, State) + end + end, + handle_missing_no_exception(Fun, Dep, State); +resolve_version(Dep, DepVsn, Hash, HexRegistry, State) -> + case valid_vsn(DepVsn) of + false -> + {error, {invalid_vsn, DepVsn}}; + _ -> + Fun = fun(Repo) -> + case resolve_version_(Dep, DepVsn, Repo, HexRegistry, State) of + none -> + not_found; + {ok, Vsn} -> + get_package(Dep, Vsn, Hash, Repo, HexRegistry, State) + end + end, + handle_missing_no_exception(Fun, Dep, State) + end. + +check_all_repos(Fun, RepoConfigs) -> + ec_lists:search(fun(#{name := R}) -> + Fun(R) + end, RepoConfigs). + +handle_missing_no_exception(Fun, Dep, State) -> + Resources = rebar_state:resources(State), + #{repos := RepoConfigs} = rebar_resource:find_resource_state(pkg, Resources), + + %% first check all repos in order for a local match + %% if none is found then we step through checking after updating the repo registry + case check_all_repos(Fun, RepoConfigs) of + not_found -> + ec_lists:search(fun(Config=#{name := R}) -> + ?MODULE:update_package(Dep, Config, State), + Fun(R) + end, RepoConfigs); + Result -> + Result + end. + +resolve_version_(Dep, DepVsn, Repo, HexRegistry, State) -> + case DepVsn of + <<"~>", Vsn/binary>> -> + highest_matching(Dep, rm_ws(Vsn), Repo, HexRegistry, State); + <<">=", Vsn/binary>> -> + cmp(Dep, rm_ws(Vsn), Repo, HexRegistry, State, fun ec_semver:gte/2); + <<">", Vsn/binary>> -> + cmp(Dep, rm_ws(Vsn), Repo, HexRegistry, State, fun ec_semver:gt/2); + <<"<=", Vsn/binary>> -> + cmpl(Dep, rm_ws(Vsn), Repo, HexRegistry, State, fun ec_semver:lte/2); + <<"<", Vsn/binary>> -> + cmpl(Dep, rm_ws(Vsn), Repo, HexRegistry, State, fun ec_semver:lt/2); + <<"==", Vsn/binary>> -> {ok, Vsn}; - {_, Vsn} -> + Vsn -> {ok, Vsn} end. @@ -308,22 +416,21 @@ valid_vsn(Vsn) -> SupportedVersions = "^(>=?|<=?|~>|==)?\\s*" ++ SemVerRegExp ++ "$", re:run(Vsn, SupportedVersions, [unicode]) =/= nomatch. -highest_matching(Dep, Vsn, HexRegistry, State) -> - case find_highest_matching_(undefined, undefined, Dep, Vsn, HexRegistry, State) of - {ok, HighestDepVsn} -> - {ok, HighestDepVsn}; - none -> - {error, {invalid_vsn, Vsn}} - end. +highest_matching(Dep, Vsn, Repo, HexRegistry, State) -> + find_highest_matching_(Dep, Vsn, #{name => Repo}, HexRegistry, State). -cmp(Dep, Vsn, HexRegistry, State, CmpFun) -> - Vsns = get_package_versions(Dep, HexRegistry, State), - cmp_(undefined, Vsn, Vsns, CmpFun). +cmp(Dep, Vsn, Repo, HexRegistry, State, CmpFun) -> + case get_package_versions(Dep, Repo, HexRegistry, State) of + [] -> + none; + Vsns -> + cmp_(undefined, Vsn, Vsns, CmpFun) + end. cmp_(undefined, MinVsn, [], _CmpFun) -> - MinVsn; + {ok, MinVsn}; cmp_(HighestDepVsn, _MinVsn, [], _CmpFun) -> - HighestDepVsn; + {ok, HighestDepVsn}; cmp_(BestMatch, MinVsn, [Vsn | R], CmpFun) -> case CmpFun(Vsn, MinVsn) of @@ -335,14 +442,18 @@ cmp_(BestMatch, MinVsn, [Vsn | R], CmpFun) -> %% We need to treat this differently since we want a version that is LOWER but %% the higest possible one. -cmpl(Dep, Vsn, HexRegistry, State, CmpFun) -> - Vsns = get_package_versions(Dep, HexRegistry, State), - cmpl_(undefined, Vsn, Vsns, CmpFun). +cmpl(Dep, Vsn, Repo, HexRegistry, State, CmpFun) -> + case get_package_versions(Dep, Repo, HexRegistry, State) of + [] -> + none; + Vsns -> + cmpl_(undefined, Vsn, Vsns, CmpFun) + end. cmpl_(undefined, MaxVsn, [], _CmpFun) -> - MaxVsn; + {ok, MaxVsn}; cmpl_(HighestDepVsn, _MaxVsn, [], _CmpFun) -> - HighestDepVsn; + {ok, HighestDepVsn}; cmpl_(undefined, MaxVsn, [Vsn | R], CmpFun) -> case CmpFun(Vsn, MaxVsn) of diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl index 5dd5956f..b8f00177 100644 --- a/src/rebar_pkg_resource.erl +++ b/src/rebar_pkg_resource.erl @@ -14,8 +14,10 @@ -export([request/4 ,etag/1]). -%% Exported for ct --export([store_etag_in_cache/2]). +-ifdef(TEST). +%% exported for test purposes +-export([store_etag_in_cache/2, repos/1, merge_repos/1]). +-endif. -include("rebar.hrl"). @@ -28,37 +30,77 @@ -type download_result() :: {bad_download, binary() | string()} | {fetch_fail, _, _} | cached_result(). +-type repo() :: #{name => unicode:unicode_binary(), + api_url => binary(), + api_key => binary(), + repo_url => binary(), + repo_public_key => binary(), + repo_verify => binary()}. + %%============================================================================== %% Public API %%============================================================================== -spec init(rebar_state:t()) -> {ok, term()}. -init(State) -> - HexConfig=#{api_url := DefaultApiUrl, - repo_url := DefaultRepoUrl, - repo_public_key := DefaultRepoPublicKey, - repo_verify := DefaultRepoVerify} = hex_core:default_config(), - ApiUrl = rebar_state:get(State, hex_api_url, DefaultApiUrl), - RepoUrl = rebar_state:get(State, hex_repo_url, - %% check legacy configuration variable for setting mirrors - rebar_state:get(State, rebar_packages_cdn, DefaultRepoUrl)), - RepoPublicKey = rebar_state:get(State, hex_repo_public_key, DefaultRepoPublicKey), - RepoVerify = rebar_state:get(State, hex_repo_verify, DefaultRepoVerify), +init(State) -> {ok, Vsn} = application:get_key(rebar, vsn), - {ok, #{hex_config => HexConfig#{api_url => ApiUrl, - repo_url => RepoUrl, - repo_public_key => RepoPublicKey, - repo_verify => RepoVerify, - http_user_agent_fragment => - <<"(rebar3/", (list_to_binary(Vsn))/binary, ") (httpc)">>, - http_adapter_config => #{profile => rebar}}}}. + BaseConfig = #{http_adapter => hex_http_httpc, + http_user_agent_fragment => + <<"(rebar3/", (list_to_binary(Vsn))/binary, ") (httpc)">>, + http_adapter_config => #{profile => rebar}}, + HexConfig = rebar_state:get(State, hex, []), + Repos = repos(HexConfig), + %% add base config entries that are specific to use by rebar3 and not overridable + Repos1 = [maps:merge(Repo, BaseConfig) || Repo <- Repos], + {ok, #{repos => Repos1, + base_config => BaseConfig}}. + +%% A user's list of repos are merged by name while keeping the order +%% intact. The order is based on the first use of a repo by name in the +%% list. The default repo is appended to the user's list. +repos(HexConfig) -> + HexDefaultConfig = default_repo(), + case [R || R <- HexConfig, element(1, R) =:= repos] of + [] -> + [HexDefaultConfig]; + %% we only care if the first element is a replace entry + [{repos, replace, Repos} | _]-> + merge_repos(Repos); + Repos -> + RepoList = repo_list(Repos), + merge_repos(RepoList ++ [HexDefaultConfig]) + end. + +-spec merge_repos([repo()]) -> [repo()]. +merge_repos(Repos) -> + lists:foldl(fun(R, ReposAcc) -> + update_repo_list(R, ReposAcc) + end, [], Repos). + +update_repo_list(R=#{name := N}, [H=#{name := HN} | Rest]) when N =:= HN -> + [maps:merge(R, H) | Rest]; +update_repo_list(R, [H | Rest]) -> + [H | update_repo_list(R, Rest)]; +update_repo_list(R, []) -> + [R]. + +default_repo() -> + HexDefaultConfig = hex_core:default_config(), + HexDefaultConfig#{name => <<"hexpm">>}. + +repo_list([]) -> + []; +repo_list([{repos, Repos} | T]) -> + Repos ++ repo_list(T); +repo_list([{repos, replace, Repos} | T]) -> + Repos ++ repo_list(T). -spec lock(AppDir, Source) -> Res when AppDir :: file:name(), Source :: tuple(), - Res :: {atom(), string(), any()}. -lock(_AppDir, Source) -> - Source. + Res :: {atom(), string(), any(), binary()}. +lock(_AppDir, {pkg, Name, Vsn, Hash, _RepoConfig}) -> + {pkg, Name, Vsn, Hash}. %%------------------------------------------------------------------------------ %% @doc @@ -68,9 +110,9 @@ lock(_AppDir, Source) -> %%------------------------------------------------------------------------------ -spec needs_update(Dir, Pkg) -> Res when Dir :: file:name(), - Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()}, + Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary(), RepoConfig :: hex_core:config()}, Res :: boolean(). -needs_update(Dir, {pkg, _Name, Vsn, _Hash}) -> +needs_update(Dir, {pkg, _Name, Vsn, _Hash, _}) -> [AppInfo] = rebar_app_discover:find_apps([Dir], all), case rebar_app_info:original_vsn(AppInfo) =:= rebar_utils:to_binary(Vsn) of true -> @@ -86,7 +128,7 @@ needs_update(Dir, {pkg, _Name, Vsn, _Hash}) -> %%------------------------------------------------------------------------------ -spec download(TmpDir, Pkg, State) -> Res when TmpDir :: file:name(), - Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()}, + Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary(), RepoConfig :: hex_core:config()}, State :: rebar_state:t(), Res :: {'error',_} | {'ok',_} | {'tarball',binary() | string()}. download(TmpDir, Pkg, State) -> @@ -101,18 +143,18 @@ download(TmpDir, Pkg, State) -> %%------------------------------------------------------------------------------ -spec download(TmpDir, Pkg, State, UpdateETag) -> Res when TmpDir :: file:name(), - Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()}, + Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary(), RepoConfig :: hex_core:config()}, State :: rebar_state:t(), UpdateETag :: boolean(), Res :: download_result(). -download(TmpDir, Pkg={pkg, Name, Vsn, _Hash}, State, UpdateETag) -> +download(TmpDir, Pkg={pkg, Name, Vsn, _Hash, _}, State, UpdateETag) -> {ok, PackageDir} = rebar_packages:package_dir(State), Package = binary_to_list(<>), ETagFile = binary_to_list(<>), CachePath = filename:join(PackageDir, Package), ETagPath = filename:join(PackageDir, ETagFile), - cached_download(TmpDir, CachePath, Pkg, etag(ETagPath), State, - ETagPath, UpdateETag). + cached_download(TmpDir, CachePath, Pkg, etag(ETagPath), + State, ETagPath, UpdateETag). %%------------------------------------------------------------------------------ %% @doc @@ -138,7 +180,7 @@ make_vsn(_) -> -> {ok, cached} | {ok, binary(), binary()} | error. request(Config, Name, Version, ETag) -> Config1 = Config#{http_etag => ETag}, - case hex_repo:get_tarball(Config1, Name, Version) of + try hex_repo:get_tarball(Config1, Name, Version) of {ok, {200, #{<<"etag">> := ETag1}, Tarball}} -> {ok, Tarball, rebar_utils:to_binary(rebar_string:trim(rebar_utils:to_list(ETag1), both, [$"]))}; {ok, {304, _Headers, _}} -> @@ -149,6 +191,10 @@ request(Config, Name, Version, ETag) -> {error, Reason} -> ?DEBUG("Request for package ~s-~s failed: ~p", [Name, Version, Reason]), error + catch + _:Exception -> + ?DEBUG("hex_repo:get_tarball failed: ~p", [Exception]), + error end. %%------------------------------------------------------------------------------ @@ -188,17 +234,15 @@ store_etag_in_cache(Path, ETag) -> UpdateETag) -> Res when TmpDir :: file:name(), CachePath :: file:name(), - Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()}, + Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary(), RepoConfig :: hex_core:config()}, ETag :: binary(), State :: rebar_state:t(), ETagPath :: file:name(), UpdateETag :: boolean(), Res :: download_result(). -cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn, _Hash}, ETag, +cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn, _Hash, RepoConfig}, ETag, State, ETagPath, UpdateETag) -> - Resources = rebar_state:resources(State), - #{hex_config := HexConfig} = rebar_resource:find_resource_state(pkg, Resources), - case request(HexConfig, Name, Vsn, ETag) of + case request(RepoConfig, Name, Vsn, ETag) of {ok, cached} -> ?INFO("Version cached at ~ts is up to date, reusing it", [CachePath]), serve_from_cache(TmpDir, CachePath, Pkg, State); @@ -218,27 +262,24 @@ cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn, _Hash}, ETag, -spec serve_from_cache(TmpDir, CachePath, Pkg, State) -> Res when TmpDir :: file:name(), CachePath :: file:name(), - Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()}, + Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary(), RepoConfig :: hex_core:config()}, State :: rebar_state:t(), Res :: cached_result(). serve_from_cache(TmpDir, CachePath, Pkg, State) -> {Files, Contents, Version, Meta} = extract(TmpDir, CachePath), case checksums(Pkg, Files, Contents, Version, Meta, State) of - {Chk, Chk, Chk, Chk} -> + {Chk, Chk, Chk} -> ok = erl_tar:extract({binary, Contents}, [{cwd, TmpDir}, compressed]), {ok, true}; - {_Hash, Chk, Chk, Chk} -> + {_Hash, Chk, Chk} -> ?DEBUG("Expected hash ~p does not match checksums ~p", [_Hash, Chk]), {unexpected_hash, CachePath, _Hash, Chk}; - {Chk, _Bin, Chk, Chk} -> + {Chk, _Bin, Chk} -> ?DEBUG("Checksums: registry: ~p, pkg: ~p", [Chk, _Bin]), {failed_extract, CachePath}; - {Chk, Chk, _Reg, Chk} -> - ?DEBUG("Checksums: registry: ~p, pkg: ~p", [_Reg, Chk]), - {bad_registry_checksum, CachePath}; - {_Hash, _Bin, _Reg, _Tar} -> - ?DEBUG("Checksums: expected: ~p, registry: ~p, pkg: ~p, meta: ~p", - [_Hash, _Reg, _Bin, _Tar]), + {_Hash, _Bin, _Tar} -> + ?DEBUG("Checksums: expected: ~p, pkg: ~p, meta: ~p", + [_Hash, _Bin, _Tar]), {bad_checksum, CachePath} end. @@ -246,7 +287,7 @@ serve_from_cache(TmpDir, CachePath, Pkg, State) -> ETagPath) -> Res when TmpDir :: file:name(), CachePath :: file:name(), - Package :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()}, + Package :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary(), RepoConfig :: hex_core:config()}, ETag :: binary(), Binary :: binary(), State :: rebar_state:t(), @@ -281,26 +322,24 @@ extract(TmpDir, CachePath) -> {Files, Contents, Version, Meta}. -spec checksums(Pkg, Files, Contents, Version, Meta, State) -> Res when - Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()}, + Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary(), RepoConfig :: hex_core:config()}, Files :: list({file:name(), binary()}), Contents :: binary(), Version :: binary(), Meta :: binary(), State :: rebar_state:t(), - Res :: {Hash, BinChecksum, RegistryChecksum, TarChecksum}, + Res :: {Hash, BinChecksum, TarChecksum}, Hash :: binary(), BinChecksum :: binary(), - RegistryChecksum :: any(), TarChecksum :: binary(). -checksums({pkg, Name, Vsn, Hash}, Files, Contents, Version, Meta, State) -> +checksums({pkg, _Name, _Vsn, Hash, _}, Files, Contents, Version, Meta, _State) -> Blob = <>, <> = crypto:hash(sha256, Blob), BinChecksum = list_to_binary( rebar_string:uppercase( lists:flatten(io_lib:format("~64.16.0b", [X])))), - RegistryChecksum = rebar_packages:registry_checksum(Name, Vsn, State), {"CHECKSUM", TarChecksum} = lists:keyfind("CHECKSUM", 1, Files), - {Hash, BinChecksum, RegistryChecksum, TarChecksum}. + {Hash, BinChecksum, TarChecksum}. -spec maybe_store_etag_in_cache(UpdateETag, Path, ETag) -> Res when UpdateETag :: boolean(), diff --git a/src/rebar_prv_install_deps.erl b/src/rebar_prv_install_deps.erl index b735ed09..4caa59ce 100644 --- a/src/rebar_prv_install_deps.erl +++ b/src/rebar_prv_install_deps.erl @@ -378,7 +378,7 @@ fetch_app(AppInfo, AppDir, State) -> Source = rebar_app_info:source(AppInfo), true = rebar_fetch:download_source(AppDir, Source, State). -format_source({pkg, Name, Vsn, _Hash}) -> {pkg, Name, Vsn}; +format_source({pkg, Name, Vsn, _Hash, _}) -> {pkg, Name, Vsn}; format_source(Source) -> Source. %% This is called after the dep has been downloaded and unpacked, if it hadn't been already. diff --git a/src/rebar_prv_packages.erl b/src/rebar_prv_packages.erl index 310d3b81..37a98a59 100644 --- a/src/rebar_prv_packages.erl +++ b/src/rebar_prv_packages.erl @@ -15,35 +15,59 @@ -spec init(rebar_state:t()) -> {ok, rebar_state:t()}. init(State) -> - State1 = rebar_state:add_provider(State, providers:create([{name, ?PROVIDER}, - {module, ?MODULE}, - {bare, true}, - {deps, ?DEPS}, - {example, "rebar3 pkgs"}, - {short_desc, "List versions of a package."}, - {desc, info("List versions of a package")}, - {opts, []}])), + State1 = rebar_state:add_provider(State, + providers:create([{name, ?PROVIDER}, + {module, ?MODULE}, + {bare, true}, + {deps, ?DEPS}, + {example, "rebar3 pkgs elli"}, + {short_desc, "List information for a package."}, + {desc, info("List information for a package")}, + {opts, [{package, undefined, undefined, string, + "Package to fetch information for."}]}])), {ok, State1}. -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. do(State) -> - Resources = rebar_state:resources(State), - #{hex_config := HexConfig} = rebar_resource:find_resource_state(pkg, Resources), - case rebar_state:command_args(State) of - [Name] -> - print_packages(rebar_packages:get(HexConfig, rebar_utils:to_binary(Name))); - _ -> - ?ERROR("Must provide a package name.", []) - end, - {ok, State}. + {Args, _} = rebar_state:command_parsed_args(State), + case proplists:get_value(package, Args, undefined) of + undefined -> + ?PRV_ERROR(no_package_arg); + Name -> + Resources = rebar_state:resources(State), + #{repos := Repos} = rebar_resource:find_resource_state(pkg, Resources), + Results = get_package(rebar_utils:to_binary(Name), Repos), + case lists:all(fun({_, {error, not_found}}) -> true; (_) -> false end, Results) of + true -> + ?PRV_ERROR({not_found, Name}); + false -> + [print_packages(Result) || Result <- Results], + {ok, State} + end + end. + +get_package(Name, Repos) -> + lists:foldl(fun(RepoConfig, Acc) -> + [{maps:get(name, RepoConfig), rebar_packages:get(RepoConfig, Name)} | Acc] + end, [], Repos). + -spec format_error(any()) -> iolist(). -format_error(load_registry_fail) -> - "Failed to load package regsitry. Try running 'rebar3 update' to fix". +format_error(no_package_arg) -> + "Missing package argument to `rebar3 pkgs` command."; +format_error({not_found, Name}) -> + io_lib:format("Package ~ts not found in any repo.", [Name]); +format_error(unknown) -> + "Something went wrong with fetching package metadata.". + -print_packages({ok, #{<<"name">> := Name, - <<"meta">> := Meta, - <<"releases">> := Releases}}) -> +print_packages({RepoName, {error, not_found}}) -> + ?CONSOLE("~ts: Package not found in this repo.~n", [RepoName]); +print_packages({RepoName, {error, _}}) -> + ?CONSOLE("~ts: Error fetching from this repo.~n", [RepoName]); +print_packages({RepoName, {ok, #{<<"name">> := Name, + <<"meta">> := Meta, + <<"releases">> := Releases}}}) -> Description = maps:get(<<"description">>, Meta, ""), Licenses = join(maps:get(<<"licenses">>, Meta, []), <<", ">>), Links = join_map(maps:get(<<"links">>, Meta, []), <<"\n ">>), @@ -51,11 +75,12 @@ print_packages({ok, #{<<"name">> := Name, Versions = [V || #{<<"version">> := V} <- Releases], VsnStr = join(Versions, <<", ">>), ?CONSOLE("~ts:~n" + " Name: ~ts~n" " Description: ~ts~n" " Licenses: ~ts~n" " Maintainers: ~ts~n" " Links:~n ~ts~n" - " Versions: ~ts~n", [Name, Description, Licenses, Maintainers, Links, VsnStr]); + " Versions: ~ts~n", [RepoName, Name, Description, Licenses, Maintainers, Links, VsnStr]); print_packages(_) -> ok. diff --git a/src/rebar_prv_repos.erl b/src/rebar_prv_repos.erl new file mode 100644 index 00000000..6f4bad33 --- /dev/null +++ b/src/rebar_prv_repos.erl @@ -0,0 +1,47 @@ +%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ts=4 sw=4 et + +-module(rebar_prv_repos). + +-behaviour(provider). + +-export([init/1, + do/1, + format_error/1]). + +-include("rebar.hrl"). + +-define(PROVIDER, repos). +-define(DEPS, []). + +%% =================================================================== +%% Public API +%% =================================================================== + +-spec init(rebar_state:t()) -> {ok, rebar_state:t()}. +init(State) -> + Provider = providers:create( + [{name, ?PROVIDER}, + {module, ?MODULE}, + {bare, false}, + {deps, ?DEPS}, + {example, "rebar3 repos"}, + {short_desc, "Print current package repository configuration"}, + {desc, "Display repository configuration for debugging purpose"}, + {opts, []}]), + State1 = rebar_state:add_provider(State, Provider), + {ok, State1}. + +-spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. +do(State) -> + Resources = rebar_state:resources(State), + #{repos := Repos} = rebar_resource:find_resource_state(pkg, Resources), + + ?CONSOLE("Repos:", []), + %%TODO: do some formatting + ?CONSOLE("~p", [Repos]), + {ok, State}. + +-spec format_error(any()) -> iolist(). +format_error(Reason) -> + io_lib:format("~p", [Reason]). diff --git a/src/rebar_prv_update.erl b/src/rebar_prv_update.erl index 35881192..b384a6c9 100644 --- a/src/rebar_prv_update.erl +++ b/src/rebar_prv_update.erl @@ -34,7 +34,11 @@ init(State) -> -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. do(State) -> Names = rebar_packages:get_all_names(State), - [rebar_packages:update_package(Name, State) || Name <- Names], + Resources = rebar_state:resources(State), + #{repos := RepoConfigs} = rebar_resource:find_resource_state(pkg, Resources), + [[update_package(Name, RepoConfig, State) + || Name <- Names] + || RepoConfig <- RepoConfigs], {ok, State}. -spec format_error(any()) -> iolist(). @@ -45,3 +49,11 @@ format_error(package_index_download) -> format_error(package_index_write) -> "Failed to write package index.". + +update_package(Name, RepoConfig, State) -> + case rebar_packages:update_package(Name, RepoConfig, State) of + fail -> + ?WARN("Failed to fetch updates for package ~ts from repo ~ts", [Name, maps:get(name, RepoConfig)]); + _ -> + ok + end. diff --git a/src/rebar_prv_upgrade.erl b/src/rebar_prv_upgrade.erl index a2664a20..82502963 100644 --- a/src/rebar_prv_upgrade.erl +++ b/src/rebar_prv_upgrade.erl @@ -134,7 +134,9 @@ update_pkg_deps([{Name, _, _} | Rest], AppInfos, State) -> {ok, AppInfo} -> case element(1, rebar_app_info:source(AppInfo)) of pkg -> - rebar_packages:update_package(Name, State); + Resources = rebar_state:resources(State), + #{repos := RepoConfigs} = rebar_resource:find_resource_state(pkg, Resources), + [update_package(Name, RepoConfig, State) || RepoConfig <- RepoConfigs]; _ -> skip end; @@ -144,6 +146,14 @@ update_pkg_deps([{Name, _, _} | Rest], AppInfos, State) -> end, update_pkg_deps(Rest, AppInfos, State). +update_package(Name, RepoConfig, State) -> + case rebar_packages:update_package(Name, RepoConfig, State) of + fail -> + ?WARN("Failed to fetch updates for package ~ts from repo ~ts", [Name, maps:get(name, RepoConfig)]); + _ -> + ok + end. + parse_names(Bin, Locks) -> case lists:usort(re:split(Bin, <<" *, *">>, [trim, unicode])) of %% Nothing submitted, use *all* apps diff --git a/src/rebar_resource.erl b/src/rebar_resource.erl index d9f35328..d0625506 100644 --- a/src/rebar_resource.erl +++ b/src/rebar_resource.erl @@ -17,7 +17,7 @@ state :: term()}). -type resource() :: #resource{}. --type source() :: {type(), location(), ref()}. +-type source() :: {type(), location(), ref()} | {type(), location(), ref(), binary()}. -type type() :: atom(). -type location() :: string(). -type ref() :: any(). diff --git a/src/rebar_state.erl b/src/rebar_state.erl index 81b9c256..9a704e86 100644 --- a/src/rebar_state.erl +++ b/src/rebar_state.erl @@ -38,6 +38,7 @@ to_list/1, + create_resources/2, resources/1, resources/2, add_resource/2, providers/1, providers/2, add_provider/2, allow_provider_overrides/1, allow_provider_overrides/2 @@ -75,26 +76,24 @@ -spec new() -> t(). new() -> - BaseState = base_state(), + BaseState = base_state(dict:new()), BaseState#state_t{dir = rebar_dir:get_cwd()}. -spec new(list()) -> t(). new(Config) when is_list(Config) -> - BaseState = base_state(), Opts = base_opts(Config), - BaseState#state_t { dir = rebar_dir:get_cwd(), - default = Opts, - opts = Opts }. + BaseState = base_state(Opts), + BaseState#state_t{dir=rebar_dir:get_cwd(), + default=Opts}. -spec new(t() | atom(), list()) -> t(). new(Profile, Config) when is_atom(Profile) , is_list(Config) -> - BaseState = base_state(), Opts = base_opts(Config), - BaseState#state_t { dir = rebar_dir:get_cwd(), - current_profiles = [Profile], - default = Opts, - opts = Opts }; + BaseState = base_state(Opts), + BaseState#state_t{dir = rebar_dir:get_cwd(), + current_profiles = [Profile], + default = Opts}; new(ParentState=#state_t{}, Config) -> %% Load terms from rebar.config, if it exists Dir = rebar_dir:get_cwd(), @@ -129,20 +128,15 @@ deps_from_config(Dir, Config) -> [{{locks, default}, D}, {{deps, default}, Deps}] end. -base_state() -> - case application:get_env(rebar, resources) of - undefined -> - Resources = []; - {ok, Resources} -> - Resources - end, - lists:foldl(fun(R, StateAcc) -> add_resource(StateAcc, R) end, #state_t{}, Resources). +base_state(Opts) -> + #state_t{opts=Opts}. base_opts(Config) -> Deps = proplists:get_value(deps, Config, []), Plugins = proplists:get_value(plugins, Config, []), ProjectPlugins = proplists:get_value(project_plugins, Config, []), - Terms = [{{deps, default}, Deps}, {{plugins, default}, Plugins}, {{project_plugins, default}, ProjectPlugins} | Config], + Terms = [{{deps, default}, Deps}, {{plugins, default}, Plugins}, + {{project_plugins, default}, ProjectPlugins} | Config], true = rebar_config:verify_config_format(Terms), dict:from_list(Terms). @@ -382,6 +376,11 @@ add_resource(State=#state_t{resources=Resources}, {ResourceType, ResourceModule} ResourceModule, ResourceState) | Resources]}. +create_resources(Resources, State) -> + lists:foldl(fun(R, StateAcc) -> + add_resource(StateAcc, R) + end, State, Resources). + providers(#state_t{providers=Providers}) -> Providers. diff --git a/src/rebar_utils.erl b/src/rebar_utils.erl index 30a23385..174de90b 100644 --- a/src/rebar_utils.erl +++ b/src/rebar_utils.erl @@ -683,7 +683,7 @@ vcs_vsn_cmd(VCS, Dir, Resources) when VCS =:= semver ; VCS =:= "semver" -> vcs_vsn_cmd({cmd, _Cmd}=Custom, _, _) -> Custom; vcs_vsn_cmd(VCS, Dir, Resources) when is_atom(VCS) -> - case find_resource_module(VCS, Resources) of + case rebar_resource:find_resource_module(VCS, Resources) of {ok, Module} -> Module:make_vsn(Dir); {error, _} -> @@ -707,19 +707,6 @@ vcs_vsn_invoke(Cmd, Dir) -> {ok, VsnString} = rebar_utils:sh(Cmd, [{cd, Dir}, {use_stdout, false}]), rebar_string:trim(VsnString, trailing, "\n"). -find_resource_module(Type, Resources) -> - case lists:keyfind(Type, 1, Resources) of - false -> - case code:which(Type) of - non_existing -> - {error, unknown}; - _ -> - {ok, Type} - end; - {Type, Module} -> - {ok, Module} - end. - %% @doc ident to the level specified -spec indent(non_neg_integer()) -> iolist(). indent(Amount) when erlang:is_integer(Amount) -> diff --git a/test/mock_git_resource.erl b/test/mock_git_resource.erl index e922af34..7b8279fc 100644 --- a/test/mock_git_resource.erl +++ b/test/mock_git_resource.erl @@ -27,7 +27,7 @@ mock(Opts) -> mock(Opts, create_app). mock(Opts, CreateType) -> - meck:new(?MOD, [no_link]), + meck:new(?MOD, [no_link, passthrough]), mock_lock(Opts), mock_update(Opts), mock_vsn(Opts), diff --git a/test/mock_pkg_resource.erl b/test/mock_pkg_resource.erl index 87319e06..333eb621 100644 --- a/test/mock_pkg_resource.erl +++ b/test/mock_pkg_resource.erl @@ -28,7 +28,7 @@ mock() -> mock([]). Vsn :: string(), Hash :: string() | undefined. mock(Opts) -> - meck:new(?MOD, [no_link]), + meck:new(?MOD, [no_link, passthrough]), mock_lock(Opts), mock_update(Opts), mock_vsn(Opts), @@ -46,7 +46,7 @@ unmock() -> %% @doc creates values for a lock file. mock_lock(_) -> - meck:expect(?MOD, lock, fun(_AppDir, Source) -> Source end). + meck:expect(?MOD, lock, fun(_AppDir, {pkg, Name, Vsn, Hash, _RepoConfig}) -> {pkg, Name, Vsn, Hash} end). %% @doc The config passed to the `mock/2' function can specify which apps %% should be updated on a per-name basis: `{update, ["App1", "App3"]}'. @@ -54,7 +54,7 @@ mock_update(Opts) -> ToUpdate = proplists:get_value(upgrade, Opts, []), meck:expect( ?MOD, needs_update, - fun(_Dir, {pkg, App, _Vsn, _Hash}) -> + fun(_Dir, {pkg, App, _Vsn, _Hash, _}) -> lists:member(binary_to_list(App), ToUpdate) end). @@ -79,7 +79,7 @@ mock_download(Opts) -> Config = proplists:get_value(config, Opts, []), meck:expect( ?MOD, download, - fun (Dir, {pkg, AppBin, Vsn, _}, _) -> + fun (Dir, {pkg, AppBin, Vsn, _, _}, _) -> App = binary_to_list(AppBin), filelib:ensure_dir(Dir), AppDeps = proplists:get_value({App,Vsn}, Deps, []), @@ -112,16 +112,18 @@ mock_download(Opts) -> %% specific applications otherwise listed. mock_pkg_index(Opts) -> Deps = proplists:get_value(pkgdeps, Opts, []), + Repos = proplists:get_value(repos, Opts, [<<"hexpm">>]), Skip = proplists:get_value(not_in_index, Opts, []), %% Dict: {App, Vsn}: [{<<"link">>, <<>>}, {<<"deps">>, []}] %% Index: all apps and deps in the index Dict = find_parts(Deps, Skip), + to_index(Deps, Dict, Repos), meck:new(rebar_packages, [passthrough, no_link]), - %% meck:expect(rebar_packages, packages, - %% fun(_State) -> to_index(Deps, Dict) end), + meck:expect(rebar_packages, update_package, + fun(_, _, _State) -> ok end), meck:expect(rebar_packages, verify_table, - fun(_State) -> to_index(Deps, Dict), true end). + fun(_State) -> true end). %%%%%%%%%%%%%%% %%% Helpers %%% @@ -149,12 +151,12 @@ parse_deps(Deps) -> [{maps:get(app, D, Name), {pkg, Name, Constraint, undefined}} || D=#{package := Name, requirement := Constraint} <- Deps]. -to_index(AllDeps, Dict) -> +to_index(AllDeps, Dict, Repos) -> catch ets:delete(?PACKAGE_TABLE), rebar_packages:new_package_table(), dict:fold( - fun(K, Deps, _) -> + fun({N, V}, Deps, _) -> DepsList = [#{package => DKB, app => DKB, requirement => DVB, @@ -162,19 +164,25 @@ to_index(AllDeps, Dict) -> || {DK, DV} <- Deps, DKB <- [ec_cnv:to_binary(DK)], DVB <- [ec_cnv:to_binary(DV)]], - ets:insert(?PACKAGE_TABLE, #package{key = K, - dependencies = parse_deps(DepsList), - checksum = <<"checksum">>}) + Repo = rebar_test_utils:random_element(Repos), + ets:insert(?PACKAGE_TABLE, #package{key={N, V, Repo}, + dependencies=parse_deps(DepsList), + retired=false, + checksum = <<"checksum">>}) end, ok, Dict), lists:foreach(fun({{Name, Vsn}, _}) -> - case ets:member(?PACKAGE_TABLE, {ec_cnv:to_binary(Name), Vsn}) of + case lists:any(fun(R) -> + ets:member(?PACKAGE_TABLE, {ec_cnv:to_binary(Name), Vsn, R}) + end, Repos) of false -> - ets:insert(?PACKAGE_TABLE, #package{key = - {ec_cnv:to_binary(Name), Vsn}, - dependencies = [], - checksum = <<"checksum">>}); + Repo = rebar_test_utils:random_element(Repos), + ets:insert(?PACKAGE_TABLE, #package{key={ec_cnv:to_binary(Name), Vsn, Repo}, + dependencies=[], + retired=false, + checksum = <<"checksum">>}); true -> ok end end, AllDeps). + diff --git a/test/rebar_deps_SUITE.erl b/test/rebar_deps_SUITE.erl index 53c08309..a011a612 100644 --- a/test/rebar_deps_SUITE.erl +++ b/test/rebar_deps_SUITE.erl @@ -229,8 +229,10 @@ deps(circular_skip) -> setup_project(Case, Config0, Deps) -> DepsType = ?config(deps_type, Config0), + %% spread packages across 3 repos randomly + Repos = [<<"test-repo-1">>, <<"test-repo-2">>, <<"hexpm">>], Config = rebar_test_utils:init_rebar_state( - Config0, + [{repos, Repos} | Config0], atom_to_list(Case)++"_"++atom_to_list(DepsType)++"_" ), AppDir = ?config(apps, Config), @@ -239,7 +241,7 @@ setup_project(Case, Config0, Deps) -> RebarConf = rebar_test_utils:create_config(AppDir, [{deps, TopDeps}]), {SrcDeps, PkgDeps} = rebar_test_utils:flat_deps(Deps), mock_git_resource:mock([{deps, SrcDeps}]), - mock_pkg_resource:mock([{pkgdeps, PkgDeps}]), + mock_pkg_resource:mock([{pkgdeps, PkgDeps}, {repos, Repos}]), [{rebarconfig, RebarConf} | Config]. mock_warnings() -> @@ -414,27 +416,27 @@ https_os_proxy_settings(_Config) -> semver_matching_lt(_Config) -> MaxVsn = <<"0.2.0">>, Vsns = [<<"0.1.7">>, <<"0.1.9">>, <<"0.1.8">>, <<"0.2.0">>, <<"0.2.1">>], - ?assertEqual(<<"0.1.9">>, + ?assertEqual({ok, <<"0.1.9">>}, rebar_packages:cmpl_(undefined, MaxVsn, Vsns, fun ec_semver:lt/2)). semver_matching_lte(_Config) -> MaxVsn = <<"0.2.0">>, Vsns = [<<"0.1.7">>, <<"0.1.9">>, <<"0.1.8">>, <<"0.2.0">>, <<"0.2.1">>], - ?assertEqual(<<"0.2.0">>, + ?assertEqual({ok, <<"0.2.0">>}, rebar_packages:cmpl_(undefined, MaxVsn, Vsns, fun ec_semver:lte/2)). semver_matching_gt(_Config) -> MaxVsn = <<"0.2.0">>, Vsns = [<<"0.1.7">>, <<"0.1.9">>, <<"0.1.8">>, <<"0.2.0">>, <<"0.2.1">>], - ?assertEqual(<<"0.2.1">>, + ?assertEqual({ok, <<"0.2.1">>}, rebar_packages:cmp_(undefined, MaxVsn, Vsns, fun ec_semver:gt/2)). semver_matching_gte(_Config) -> MaxVsn = <<"0.2.0">>, Vsns = [<<"0.1.7">>, <<"0.1.9">>, <<"0.1.8">>, <<"0.2.0">>], - ?assertEqual(<<"0.2.0">>, + ?assertEqual({ok, <<"0.2.0">>}, rebar_packages:cmp_(undefined, MaxVsn, Vsns, fun ec_semver:gt/2)). @@ -496,5 +498,5 @@ in_warnings(git, Warns, NameRaw, VsnRaw) -> in_warnings(pkg, Warns, NameRaw, VsnRaw) -> Name = iolist_to_binary(NameRaw), Vsn = iolist_to_binary(VsnRaw), - 1 =< length([1 || {_, [AppName, {pkg, _, AppVsn, _}]} <- Warns, + 1 =< length([1 || {_, [AppName, {pkg, _, AppVsn, _, _}]} <- Warns, AppName =:= Name, AppVsn =:= Vsn]). diff --git a/test/rebar_install_deps_SUITE.erl b/test/rebar_install_deps_SUITE.erl index dccb7e06..93f5fafc 100644 --- a/test/rebar_install_deps_SUITE.erl +++ b/test/rebar_install_deps_SUITE.erl @@ -475,5 +475,5 @@ in_warnings(git, Warns, NameRaw, VsnRaw) -> in_warnings(pkg, Warns, NameRaw, VsnRaw) -> Name = iolist_to_binary(NameRaw), Vsn = iolist_to_binary(VsnRaw), - 1 =< length([1 || {_, [AppName, {pkg, _, AppVsn, _}]} <- Warns, + 1 =< length([1 || {_, [AppName, {pkg, _, AppVsn, _, _}]} <- Warns, AppName =:= Name, AppVsn =:= Vsn]). diff --git a/test/rebar_localfs_resource.erl b/test/rebar_localfs_resource.erl index d60421e9..63687c00 100644 --- a/test/rebar_localfs_resource.erl +++ b/test/rebar_localfs_resource.erl @@ -13,13 +13,18 @@ -behaviour(rebar_resource). --export([lock/2 +-export([init/1 + ,lock/2 ,download/3 ,needs_update/2 ,make_vsn/1]). -include_lib("eunit/include/eunit.hrl"). +-spec init(rebar_state:t()) -> {ok, term()}. +init(_State) -> + {ok, #{}}. + lock(AppDir, {localfs, Path, _Ref}) -> lock(AppDir, {localfs, Path}); lock(_AppDir, {localfs, Path}) -> diff --git a/test/rebar_pkg_SUITE.erl b/test/rebar_pkg_SUITE.erl index e8546c23..605d0dad 100644 --- a/test/rebar_pkg_SUITE.erl +++ b/test/rebar_pkg_SUITE.erl @@ -12,8 +12,8 @@ -define(good_checksum, <<"1C6CE379D191FBAB41B7905075E0BF87CBBE23C77CECE775C5A0B786B2244C35">>). -define(BADPKG_ETAG, <<"BADETAG">>). -all() -> [good_uncached, good_cached, badindexchk, badpkg, - badhash_nocache, badhash_cache, +all() -> [good_uncached, good_cached, badpkg, + %% badindexchk, badhash_nocache, badhash_cache, bad_to_good, good_disconnect, bad_disconnect, pkgs_provider, find_highest_matching]. @@ -122,7 +122,7 @@ good_uncached(Config) -> {Pkg,Vsn} = ?config(pkg, Config), State = ?config(state, Config), ?assertEqual({ok, true}, - rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum}, State)), + rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum, #{}}, State)), Cache = ?config(cache_dir, Config), ?assert(filelib:is_regular(filename:join(Cache, <>))). @@ -135,19 +135,9 @@ good_cached(Config) -> ?assert(filelib:is_regular(CachedFile)), {ok, Content} = file:read_file(CachedFile), ?assertEqual({ok, true}, - rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum}, State)), + rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum, #{}}, State)), {ok, Content} = file:read_file(CachedFile). -badindexchk(Config) -> - Tmp = ?config(tmp_dir, Config), - {Pkg,Vsn} = ?config(pkg, Config), - State = ?config(state, Config), - ?assertMatch({bad_registry_checksum, _Path}, - rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum}, State)), - %% The cached file is there for forensic purposes - Cache = ?config(cache_dir, Config), - ?assert(filelib:is_regular(filename:join(Cache, <>))). - badpkg(Config) -> Tmp = ?config(tmp_dir, Config), {Pkg,Vsn} = ?config(pkg, Config), @@ -157,35 +147,11 @@ badpkg(Config) -> ETagPath = filename:join(Cache, <>), rebar_pkg_resource:store_etag_in_cache(ETagPath, ?BADPKG_ETAG), ?assertMatch({bad_download, _Path}, - rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum}, State, false)), + rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum, #{}}, State, false)), %% The cached/etag files are there for forensic purposes ?assert(filelib:is_regular(ETagPath)), ?assert(filelib:is_regular(CachePath)). -badhash_nocache(Config) -> - Tmp = ?config(tmp_dir, Config), - {Pkg,Vsn} = ?config(pkg, Config), - State = ?config(state, Config), - ?assertMatch({unexpected_hash, _Path, ?bad_checksum, ?good_checksum}, - rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?bad_checksum}, State)), - %% The cached file is there for forensic purposes - Cache = ?config(cache_dir, Config), - ?assert(filelib:is_regular(filename:join(Cache, <>))). - -badhash_cache(Config) -> - Tmp = ?config(tmp_dir, Config), - {Pkg,Vsn} = ?config(pkg, Config), - Cache = ?config(cache_dir, Config), - State = ?config(state, Config), - CachedFile = filename:join(Cache, <>), - ?assert(filelib:is_regular(CachedFile)), - {ok, Content} = file:read_file(CachedFile), - ?assertMatch({unexpected_hash, _Path, ?bad_checksum, ?good_checksum}, - rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?bad_checksum}, State)), - %% The cached file is there still, unchanged. - ?assert(filelib:is_regular(CachedFile)), - ?assertEqual({ok, Content}, file:read_file(CachedFile)). - bad_to_good(Config) -> Tmp = ?config(tmp_dir, Config), {Pkg,Vsn} = ?config(pkg, Config), @@ -195,7 +161,7 @@ bad_to_good(Config) -> ?assert(filelib:is_regular(CachedFile)), {ok, Contents} = file:read_file(CachedFile), ?assertEqual({ok, true}, - rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum}, State)), + rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum, #{}}, State)), %% Cache has refreshed ?assert({ok, Contents} =/= file:read_file(CachedFile)). @@ -210,7 +176,7 @@ good_disconnect(Config) -> {ok, Content} = file:read_file(CachedFile), rebar_pkg_resource:store_etag_in_cache(ETagFile, ?BADPKG_ETAG), ?assertEqual({ok, true}, - rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum}, State)), + rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum, #{}}, State)), {ok, Content} = file:read_file(CachedFile). bad_disconnect(Config) -> @@ -218,32 +184,31 @@ bad_disconnect(Config) -> {Pkg,Vsn} = ?config(pkg, Config), State = ?config(state, Config), ?assertEqual({fetch_fail, Pkg, Vsn}, - rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum}, State)). + rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum, #{}}, State)). pkgs_provider(Config) -> Config1 = rebar_test_utils:init_rebar_state(Config), rebar_test_utils:run_and_check( - Config1, [], ["pkgs"], + Config1, [], ["pkgs", "relx"], {ok, []} ). find_highest_matching(_Config) -> State = rebar_state:new(), - {ok, Vsn} = rebar_packages:find_highest_matching( - <<"test">>, <<"1.0.0">>, <<"goodpkg">>, <<"1.0.0">>, ?PACKAGE_TABLE, State), + {ok, Vsn} = rebar_packages:find_highest_matching_( + <<"goodpkg">>, <<"1.0.0">>, #{name => <<"hexpm">>}, ?PACKAGE_TABLE, State), ?assertEqual(<<"1.0.1">>, Vsn), {ok, Vsn1} = rebar_packages:find_highest_matching( - <<"test">>, <<"1.0.0">>, <<"goodpkg">>, <<"1.0">>, ?PACKAGE_TABLE, State), + <<"goodpkg">>, <<"1.0">>, #{name => <<"hexpm">>}, ?PACKAGE_TABLE, State), ?assertEqual(<<"1.1.1">>, Vsn1), {ok, Vsn2} = rebar_packages:find_highest_matching( - <<"test">>, <<"1.0.0">>, <<"goodpkg">>, <<"2.0">>, ?PACKAGE_TABLE, State), + <<"goodpkg">>, <<"2.0">>, #{name => <<"hexpm">>}, ?PACKAGE_TABLE, State), ?assertEqual(<<"2.0.0">>, Vsn2), %% regression test. ~> constraints higher than the available packages would result %% in returning the first package version instead of 'none'. - ?assertEqual(none, rebar_packages:find_highest_matching(<<"test">>, <<"1.0.0">>, <<"goodpkg">>, - <<"~> 5.0">>, ?PACKAGE_TABLE, State)). - + ?assertEqual(none, rebar_packages:find_highest_matching_(<<"goodpkg">>, <<"~> 5.0">>, + #{name => <<"hexpm">>}, ?PACKAGE_TABLE, State)). %%%%%%%%%%%%%%% %%% Helpers %%% @@ -269,12 +234,12 @@ mock_config(Name, Config) -> catch ets:delete(?PACKAGE_TABLE), rebar_packages:new_package_table(), lists:foreach(fun({{N, Vsn}, [Deps, Checksum, _]}) -> - case ets:member(?PACKAGE_TABLE, {ec_cnv:to_binary(N), Vsn}) of + case ets:member(?PACKAGE_TABLE, {ec_cnv:to_binary(N), Vsn, <<"hexpm">>}) of false -> - ets:insert(?PACKAGE_TABLE, #package{key = - {ec_cnv:to_binary(N), Vsn}, - dependencies = Deps, - checksum = Checksum}); + ets:insert(?PACKAGE_TABLE, #package{key={ec_cnv:to_binary(N), Vsn, <<"hexpm">>}, + dependencies=Deps, + retired=false, + checksum=Checksum}); true -> ok end @@ -303,8 +268,10 @@ mock_config(Name, Config) -> end), meck:expect(rebar_state, resources, fun(_State) -> - [rebar_resource:new(pkg, rebar_pkg_resource, - #{hex_config => hex_core:default_config()})] + DefaultConfig = hex_core:default_config(), + [rebar_resource:new(pkg, rebar_pkg_resource, + #{repos => [DefaultConfig#{name => <<"hexpm">>}], + base_config => #{}})] end), meck:new(rebar_dir, [passthrough]), diff --git a/test/rebar_pkg_alias_SUITE.erl b/test/rebar_pkg_alias_SUITE.erl index 49cc471a..0dd829cb 100644 --- a/test/rebar_pkg_alias_SUITE.erl +++ b/test/rebar_pkg_alias_SUITE.erl @@ -217,6 +217,9 @@ mock_config(Name, Config) -> meck:expect(rebar_packages, registry_dir, fun(_) -> {ok, CacheDir} end), meck:expect(rebar_packages, package_dir, fun(_) -> {ok, CacheDir} end), + %% TODO: is something else wrong that we need this for transitive_alias to pass + meck:expect(rebar_packages, update_package, fun(_, _, _) -> ok end), + meck:new(rebar_prv_update, [passthrough]), meck:expect(rebar_prv_update, do, fun(State) -> {ok, State} end), @@ -224,13 +227,12 @@ mock_config(Name, Config) -> rebar_packages:new_package_table(), lists:foreach(fun({{N, Vsn}, [Deps, Checksum, _]}) -> - case ets:member(?PACKAGE_TABLE, {ec_cnv:to_binary(N), Vsn}) of + case ets:member(?PACKAGE_TABLE, {ec_cnv:to_binary(N), Vsn, <<"hexpm">>}) of false -> - ets:insert(?PACKAGE_TABLE, #package{key = - {ec_cnv:to_binary(N), Vsn}, - dependencies = [{DAppName, {pkg, DN, DV, undefined}} || {DN, DV, _, DAppName} <- Deps], - - checksum = Checksum}); + ets:insert(?PACKAGE_TABLE, #package{key={ec_cnv:to_binary(N), Vsn, <<"hexpm">>}, + dependencies=[{DAppName, {pkg, DN, DV, undefined}} || {DN, DV, _, DAppName} <- Deps], + retired=false, + checksum=Checksum}); true -> ok end; @@ -240,7 +242,7 @@ mock_config(Name, Config) -> end, AllDeps), meck:expect(rebar_packages, registry_checksum, - fun(N, V, _) -> + fun(N, V, _, _) -> case ets:match_object(Tid, {{N, V}, '_'}) of [{{_, _}, [_, Checksum, _]}] -> Checksum @@ -257,7 +259,7 @@ mock_config(Name, Config) -> Releases = [#{checksum => Checksum, version => Vsn, - dependencies => [{DAppName, {pkg, DN, DV, undefined}} || {DN, DV, _, DAppName} <- Deps]} || + dependencies => [{DAppName, {pkg, DN, DV, undefined}} || {DN, DV, _, DAppName} <- Deps]} || {{_, Vsn}, [Deps, Checksum, _]} <- Matches], {ok, {200, #{}, #{releases => Releases}}}%% ; %% _ -> diff --git a/test/rebar_pkg_repos_SUITE.erl b/test/rebar_pkg_repos_SUITE.erl new file mode 100644 index 00000000..97e2e3c0 --- /dev/null +++ b/test/rebar_pkg_repos_SUITE.erl @@ -0,0 +1,257 @@ +%% Test suite for the handling hexpm repo configurations +-module(rebar_pkg_repos_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include("rebar.hrl"). + +all() -> + [default_repo, repo_merging, repo_replacing, {group, resolve_version}]. + +groups() -> + [{resolve_version, [use_first_repo_match, use_exact_with_hash, fail_repo_update, + ignore_match_in_excluded_repo]}]. + +init_per_group(resolve_version, Config) -> + Repo1 = <<"test-repo-1">>, + Repo2 = <<"test-repo-2">>, + Repo3 = <<"test-repo-3">>, + Hexpm = <<"hexpm">>, + Repos = [Repo1, Repo2, Repo3, Hexpm], + + Deps = [{"A", "0.1.1", <<"good checksum">>, Repo1}, + {"A", "0.1.1", <<"good checksum">>, Repo2}, + {"B", "1.0.0", Repo1}, + {"B", "2.0.0", Repo2}, + {"B", "1.4.0", Repo3}, + {"B", "1.4.3", Hexpm}, + {"C", "1.3.1", <<"bad checksum">>, Repo1}, + {"C", "1.3.1", <<"good checksum">>, Repo2}], + [{deps, Deps}, {repos, Repos} | Config]; +init_per_group(_, Config) -> + Config. + +end_per_group(_, _) -> + ok. + +init_per_testcase(use_first_repo_match, Config) -> + Deps = ?config(deps, Config), + Repos = ?config(repos, Config), + State = setup_deps_and_repos(Deps, Repos), + + meck:new(rebar_packages, [passthrough, no_link]), + + %% fail when the first repo is updated since it doesn't have a matching package + %% should continue anyway + meck:expect(rebar_packages, update_package, + fun(_, _, _State) -> ok end), + meck:expect(rebar_packages, verify_table, + fun(_State) -> true end), + + [{state, State} | Config]; +init_per_testcase(use_exact_with_hash, Config) -> + Deps = ?config(deps, Config), + Repos = ?config(repos, Config), + State = setup_deps_and_repos(Deps, Repos), + + meck:new(rebar_packages, [passthrough, no_link]), + + %% fail when the first repo is updated since it doesn't have a matching package + %% should continue anyway + meck:expect(rebar_packages, update_package, + fun(_, _, _State) -> ok end), + meck:expect(rebar_packages, verify_table, + fun(_State) -> true end), + + [{state, State} | Config]; +init_per_testcase(fail_repo_update, Config) -> + Deps = ?config(deps, Config), + Repos = ?config(repos, Config), + State = setup_deps_and_repos(Deps, Repos), + + meck:new(rebar_packages, [passthrough, no_link]), + + %% fail when the first repo is updated since it doesn't have a matching package + %% should continue anyway + [Repo1 | _] = Repos, + meck:expect(rebar_packages, update_package, + fun(_, #{name := Repo}, _State) when Repo =:= Repo1 -> fail; + (_, _, _State) -> ok end), + meck:expect(rebar_packages, verify_table, + fun(_State) -> true end), + + [{state, State} | Config]; +init_per_testcase(ignore_match_in_excluded_repo, Config) -> + Deps = ?config(deps, Config), + Repos = [Repo1, _, Repo3 | _] = ?config(repos, Config), + + %% drop repo1 and repo2 from the repos to be used by the pkg resource + State = setup_deps_and_repos(Deps, [R || R <- Repos, R =/= Repo3, R =/= Repo1]), + + meck:new(rebar_packages, [passthrough, no_link]), + + %% fail when the first repo is updated since it doesn't have a matching package + %% should continue anyway + [_, _, Repo3 | _] = Repos, + meck:expect(rebar_packages, update_package, + fun(_, _, _State) -> ok end), + meck:expect(rebar_packages, verify_table, + fun(_State) -> true end), + + [{state, State} | Config]; +init_per_testcase(_, Config) -> + Config. + +end_per_testcase(Case, _Config) when Case =:= use_first_repo_match ; + Case =:= use_exact_with_hash ; + Case =:= fail_repo_update ; + Case =:= ignore_match_in_excluded_repo -> + meck:unload(rebar_packages); +end_per_testcase(_, _) -> + ok. + + +default_repo(_Config) -> + Repo1 = #{name => <<"hexpm">>, + api_key => <<"asdf">>}, + + MergedRepos = rebar_pkg_resource:repos([{repos, [Repo1]}]), + + ?assertMatch([#{name := <<"hexpm">>, + api_key := <<"asdf">>, + api_url := <<"https://hex.pm/api">>}], MergedRepos). + + +repo_merging(_Config) -> + Repo1 = #{name => <<"repo-1">>, + api_url => <<"repo-1/api">>}, + Repo2 = #{name => <<"repo-2">>, + repo_url => <<"repo-2/repo">>, + repo_verify => false}, + Result = rebar_pkg_resource:merge_repos([Repo1, Repo2, + #{name => <<"repo-2">>, + api_url => <<"repo-2/api">>, + repo_url => <<"bad url">>, + repo_verify => true}, + #{name => <<"repo-1">>, + api_url => <<"bad url">>, + repo_verify => true}, + #{name => <<"repo-2">>, + organization => <<"repo-2-org">>, + api_url => <<"repo-2/api-2">>, + repo_url => <<"other/repo">>}]), + ?assertMatch([#{name := <<"repo-1">>, + api_url := <<"repo-1/api">>, + repo_verify := true}, + #{name := <<"repo-2">>, + api_url := <<"repo-2/api">>, + repo_url := <<"repo-2/repo">>, + organization := <<"repo-2-org">>, + repo_verify := false}], Result). + +repo_replacing(_Config) -> + Repo1 = #{name => <<"repo-1">>, + api_url => <<"repo-1/api">>}, + Repo2 = #{name => <<"repo-2">>, + repo_url => <<"repo-2/repo">>, + repo_verify => false}, + + ?assertMatch([Repo1, Repo2, #{name := <<"hexpm">>}], + rebar_pkg_resource:repos([{repos, [Repo1]}, + {repos, [Repo2]}])), + + %% use of replace is ignored if found in later entries than the first + ?assertMatch([Repo1, Repo2, #{name := <<"hexpm">>}], + rebar_pkg_resource:repos([{repos, [Repo1]}, + {repos, replace, [Repo2]}])), + + ?assertMatch([Repo1], + rebar_pkg_resource:repos([{repos, replace, [Repo1]}, + {repos, [Repo2]}])). + + + +use_first_repo_match(Config) -> + State = ?config(state, Config), + + ?assertMatch({ok,{package,{<<"B">>, <<"2.0.0">>, Repo2}, + <<"some checksum">>, false, []}, + #{name := Repo2, + http_adapter_config := #{profile := rebar}}}, + rebar_packages:resolve_version(<<"B">>, <<"> 1.4.0">>, undefined, + ?PACKAGE_TABLE, State)), + + ?assertMatch({ok,{package,{<<"B">>, <<"1.4.0">>, Repo3}, + <<"some checksum">>, false, []}, + #{name := Repo3, + http_adapter_config := #{profile := rebar}}}, + rebar_packages:resolve_version(<<"B">>, <<"~> 1.4.0">>, undefined, + ?PACKAGE_TABLE, State)). + +%% tests that even though an easier repo has C-1.3.1 it doesn't use it since its hash is different +use_exact_with_hash(Config) -> + State = ?config(state, Config), + + ?assertMatch({ok,{package,{<<"C">>, <<"1.3.1">>, Repo2}, + <<"good checksum">>, false, []}, + #{name := Repo2, + http_adapter_config := #{profile := rebar}}}, + rebar_packages:resolve_version(<<"C">>, <<"1.3.1">>, <<"good checksum">>, + ?PACKAGE_TABLE, State)). + +fail_repo_update(Config) -> + State = ?config(state, Config), + + ?assertMatch({ok,{package,{<<"B">>, <<"1.4.0">>, Repo3}, + <<"some checksum">>, false, []}, + #{name := Repo3, + http_adapter_config := #{profile := rebar}}}, + rebar_packages:resolve_version(<<"B">>, <<"~> 1.4.0">>, undefined, + ?PACKAGE_TABLE, State)). + +ignore_match_in_excluded_repo(Config) -> + State = ?config(state, Config), + Repos = ?config(repos, Config), + + ?assertMatch({ok,{package,{<<"B">>, <<"1.4.3">>, Hexpm}, + <<"some checksum">>, false, []}, + #{name := Hexpm, + http_adapter_config := #{profile := rebar}}}, + rebar_packages:resolve_version(<<"B">>, <<"~> 1.4.0">>, undefined, + ?PACKAGE_TABLE, State)), + + [_, Repo2 | _] = Repos, + ?assertMatch({ok,{package,{<<"A">>, <<"0.1.1">>, Repo2}, + <<"good checksum">>, false, []}, + #{name := Repo2, + http_adapter_config := #{profile := rebar}}}, + rebar_packages:resolve_version(<<"A">>, <<"0.1.1">>, <<"good checksum">>, + ?PACKAGE_TABLE, State)). + +%% + +setup_deps_and_repos(Deps, Repos) -> + true = rebar_packages:new_package_table(), + insert_deps(Deps), + State = rebar_state:new([{hex, [{repos, [#{name => R} || R <- Repos]}]}]), + rebar_state:create_resources([{pkg, rebar_pkg_resource}], State). + + +insert_deps(Deps) -> + lists:foreach(fun({Name, Version, Repo}) -> + ets:insert(?PACKAGE_TABLE, #package{key={rebar_utils:to_binary(Name), + rebar_utils:to_binary(Version), + rebar_utils:to_binary(Repo)}, + dependencies=[], + retired=false, + checksum = <<"some checksum">>}); + ({Name, Version, Checksum, Repo}) -> + ets:insert(?PACKAGE_TABLE, #package{key={rebar_utils:to_binary(Name), + rebar_utils:to_binary(Version), + rebar_utils:to_binary(Repo)}, + dependencies=[], + retired=false, + checksum = Checksum}) + end, Deps). diff --git a/test/rebar_test_utils.erl b/test/rebar_test_utils.erl index b74aa2f6..7de4899b 100644 --- a/test/rebar_test_utils.erl +++ b/test/rebar_test_utils.erl @@ -5,7 +5,8 @@ -export([expand_deps/2, flat_deps/1, top_level_deps/1]). -export([create_app/4, create_plugin/4, create_eunit_app/4, create_empty_app/4, create_config/2, create_config/3, package_app/3]). --export([create_random_name/1, create_random_vsn/0, write_src_file/2]). +-export([create_random_name/1, create_random_vsn/0, write_src_file/2, + random_element/1]). %% Pick the right random module -ifdef(rand_only). @@ -34,8 +35,10 @@ init_rebar_state(Config, Name) -> Verbosity = rebar3:log_level(), rebar_log:init(command_line, Verbosity), GlobalDir = filename:join([DataDir, "cache"]), + Repos = proplists:get_value(repos, Config, []), State = rebar_state:new([{base_dir, filename:join([AppsDir, "_build"])} ,{global_rebar_dir, GlobalDir} + ,{hex, [{repos, [#{name => R} || R <- Repos]}]} ,{root_dir, AppsDir}]), [{apps, AppsDir}, {checkouts, CheckoutsDir}, {state, State} | Config]. @@ -488,3 +491,7 @@ package_app(AppDir, DestDir, PkgName) -> <> = crypto:hash(md5, BinFull), Etag = rebar_string:lowercase(lists:flatten(io_lib:format("~32.16.0b", [E]))), {BinChecksum, Etag}. + +random_element(Repos) -> + Index = ?random:uniform(length(Repos)), + lists:nth(Index, Repos).