warn if the matched package is retired, skip prerelease (#1897)

retired packages are now used the same as any other but
a warning will be printed when it is resolved.

prerelease versions are skipped unless explicitly given
as the version in the constraint or lock file.

src/rebar.hrl

@@ -27,9 +27,9 @@
 -define(REMOTE_PACKAGE_DIR, "tarballs").
 -define(LOCK_FILE, "rebar.lock").
 -define(DEFAULT_COMPILER_SOURCE_FORMAT, relative).
--define(PACKAGE_INDEX_VERSION, 4).
--define(PACKAGE_TABLE, package_index_v4).
--define(INDEX_FILE, "packages-v4.idx").
+-define(PACKAGE_INDEX_VERSION, 5).
+-define(PACKAGE_TABLE, package_index).
+-define(INDEX_FILE, "packages.idx").
 -define(HEX_AUTH_FILE, "hex.config").
 -define(PUBLIC_HEX_REPO, <<"hexpm">>).
 

src/rebar_app_utils.erl

@@ -254,10 +254,13 @@ update_source(AppInfo, {pkg, PkgName, PkgVsn, Hash}, State) ->
         {ok, Package, RepoConfig} ->
             #package{key={_, PkgVsn1, _},
                      checksum=Hash1,
-                     dependencies=Deps} = Package,
-            AppInfo1 = rebar_app_info:source(AppInfo, {pkg, PkgName, PkgVsn1, Hash1, RepoConfig}),
+                     dependencies=Deps,
+                     retired=Retired} = Package,
+            maybe_warn_retired(PkgName, PkgVsn1, Hash, Retired),
+            PkgVsn2 = list_to_binary(lists:flatten(ec_semver:format(PkgVsn1))),
+            AppInfo1 = rebar_app_info:source(AppInfo, {pkg, PkgName, PkgVsn2, Hash1, RepoConfig}),
             AppInfo2 = rebar_app_info:update_opts_deps(AppInfo1, Deps),
-            rebar_app_info:original_vsn(AppInfo2, PkgVsn1);
+            rebar_app_info:original_vsn(AppInfo2, PkgVsn2);
         not_found ->
             throw(?PRV_ERROR({missing_package, PkgName, PkgVsn}));
         {error, {invalid_vsn, InvalidVsn}} ->
@@ -269,10 +272,10 @@ update_source(AppInfo, Source, _State) ->
 %% @doc convert a given exception's payload into an io description.
 -spec format_error(any()) -> iolist().
 format_error({missing_package, Name, undefined}) ->
-    io_lib:format("Package not found in any repo: ~ts.", [rebar_utils:to_binary(Name)]);
+    io_lib:format("Package not found in any repo: ~ts", [rebar_utils:to_binary(Name)]);
 format_error({missing_package, Name, Vsn}) ->
-    io_lib:format("Package not found in any repo: ~ts-~ts.", [rebar_utils:to_binary(Name),
-                                                              rebar_utils:to_binary(Vsn)]);
+    io_lib:format("Package not found in any repo: ~ts ~ts", [rebar_utils:to_binary(Name),
+                                                             rebar_utils:to_binary(Vsn)]);
 format_error({parse_dep, Dep}) ->
     io_lib:format("Failed parsing dep ~p", [Dep]);
 format_error({invalid_vsn, Dep, InvalidVsn}) ->
@@ -284,6 +287,32 @@ format_error(Error) ->
 %% Internal functions
 %% ===================================================================
 
+maybe_warn_retired(_, _, _, false) ->
+    ok;
+maybe_warn_retired(_, _, Hash, _) when is_binary(Hash) ->
+    %% don't warn if this is a lock
+    ok;
+maybe_warn_retired(Name, Vsn, _, R=#{reason := Reason}) ->
+    Message = maps:get(message, R, ""),
+    ?WARN("Warning: package ~s-~s is retired: (~s) ~s",
+          [Name, ec_semver:format(Vsn), retire_reason(Reason), Message]);
+maybe_warn_retired(_, _, _, _) ->
+    ok.
+
+%% TODO: move to hex_core
+retire_reason('RETIRED_OTHER') ->
+    "other";
+retire_reason('RETIRED_INVALID') ->
+    "invalid";
+retire_reason('RETIRED_SECURITY') ->
+    "security";
+retire_reason('RETIRED_DEPRECATED') ->
+    "deprecated";
+retire_reason('RETIRED_RENAMED') ->
+    "renamed";
+retire_reason(_Other) ->
+    "other".
+
 %% @private checks that all the beam files have been properly
 %% created.
 -spec has_all_beams(file:filename_all(), [module()]) ->

src/rebar_packages.erl

@@ -24,10 +24,10 @@
 -type package() :: pkg_name() | {pkg_name(), vsn()}.
 
 format_error({missing_package, Name, Vsn}) ->
-    io_lib:format("Package not found in any repo: ~ts-~ts.", [rebar_utils:to_binary(Name),
-                                                              rebar_utils:to_binary(Vsn)]);
+    io_lib:format("Package not found in any repo: ~ts ~ts", [rebar_utils:to_binary(Name),
+                                                             rebar_utils:to_binary(Vsn)]);
 format_error({missing_package, Pkg}) ->
-    io_lib:format("Package not found in any repo: ~p.", [Pkg]).
+    io_lib:format("Package not found in any repo: ~p", [Pkg]).
 
 -spec get(rebar_hex_repos:repo(), binary()) -> {ok, map()} | {error, term()}.
 get(Config, Name) ->
@@ -55,29 +55,25 @@ get_all_names(State) ->
                                                       _='_'}, 
                                              [], ['$1']}])).
 
--spec get_package_versions(unicode:unicode_binary(), unicode:unicode_binary(), ets:tid(), rebar_state:t())
-                          -> [vsn()].
+-spec get_package_versions(unicode:unicode_binary(), unicode:unicode_binary(),
+                           ets:tid(), rebar_state:t()) -> [vsn()].
 get_package_versions(Dep, Repo, Table, State) ->
     ?MODULE:verify_table(State),
-    ets:select(Table, [{#package{key={Dep,'$1', Repo},
-                                 _='_'}, 
-                        [], ['$1']}]).
-
-
-get_package(Dep, Vsn, Hash, Repo, Table, State) ->
-    get_package(Dep, Vsn, Hash, false, [Repo], Table, State).
+    AllowPreRelease = rebar_state:get(State, deps_allow_prerelease, false),
+    ets:select(Table, [{#package{key={Dep,{'$1', '$2'}, Repo},
+                                 _='_'},
+                        [{'==', '$2', {{[],[]}}} || not AllowPreRelease], [{{'$1', '$2'}}]}]).
 
 -spec get_package(unicode:unicode_binary(), unicode:unicode_binary(),
-                  binary() | undefined | '_', boolean() | '_',
-                  unicode:unicode_binary() | '_' | list(), ets:tab(), rebar_state:t())
+                  binary() | undefined | '_',
+                  [unicode:unicode_binary()] | ['_'], ets:tab(), rebar_state:t())
                  -> {ok, #package{}} | not_found.
-get_package(Dep, Vsn, undefined, Retired, Repo, Table, State) ->
-    get_package(Dep, Vsn, '_', Retired, Repo, Table, State);
-get_package(Dep, Vsn, Hash, Retired, Repos, Table, State) ->
+get_package(Dep, Vsn, undefined, Repos, Table, State) ->
+    get_package(Dep, Vsn, '_', Repos, Table, State);
+get_package(Dep, Vsn, Hash, Repos, Table, State) ->
     ?MODULE:verify_table(State),
-    case ets:select(Table, [{#package{key={Dep, Vsn, Repo},
+    case ets:select(Table, [{#package{key={Dep, ec_semver:parse(Vsn), Repo},
                                       checksum=Hash,
-                                      retired=Retired,
                                       _='_'}, [], ['$_']} || Repo <- Repos]) of
         %% have to allow multiple matches in the list for cases that Repo is `_`
         [Package | _] ->
@@ -258,7 +254,7 @@ update_package(Name, RepoConfig=#{name := Repo}, State) ->
 
 insert_releases(Name, Releases, Repo, Table) ->
     [true = ets:insert(Table,
-                       #package{key={Name, Version, Repo},
+                       #package{key={Name, ec_semver:parse(Version), Repo},
                                 checksum=parse_checksum(Checksum),
                                 retired=maps:get(retired, Release, false),
                                 dependencies=parse_deps(Dependencies)})
@@ -279,7 +275,7 @@ resolve_version(Dep, DepVsn, Hash, HexRegistry, State) when is_binary(Hash) ->
     RepoNames = [RepoName || #{name := RepoName} <- RepoConfigs],
 
     %% allow retired packages when we have a checksum
-    case get_package(Dep, DepVsn, Hash, '_', RepoNames, HexRegistry, State) of
+    case get_package(Dep, DepVsn, Hash, RepoNames, HexRegistry, State) of
         {ok, Package=#package{key={_, _, RepoName}}} ->
             {ok, RepoConfig} = rebar_hex_repos:get_repo_config(RepoName, RepoConfigs),
             {ok, Package, RepoConfig};
@@ -289,7 +285,7 @@ resolve_version(Dep, DepVsn, Hash, HexRegistry, State) when is_binary(Hash) ->
                           none ->
                               not_found;
                           {ok, Vsn} ->
-                              get_package(Dep, Vsn, Hash, Repo, HexRegistry, State)
+                              get_package(Dep, Vsn, Hash, [Repo], HexRegistry, State)
                       end
                   end,
             handle_missing_no_exception(Fun, Dep, State)
@@ -300,7 +296,7 @@ resolve_version(Dep, undefined, Hash, HexRegistry, State) ->
                   none ->
                       not_found;
                   {ok, Vsn} ->
-                      get_package(Dep, Vsn, Hash, Repo, HexRegistry, State)
+                      get_package(Dep, Vsn, Hash, [Repo], HexRegistry, State)
               end
           end,
     handle_missing_no_exception(Fun, Dep, State);
@@ -314,7 +310,7 @@ resolve_version(Dep, DepVsn, Hash, HexRegistry, State) ->
                           none ->
                               not_found;
                           {ok, Vsn} ->
-                              get_package(Dep, Vsn, Hash, Repo, HexRegistry, State)
+                              get_package(Dep, Vsn, Hash, [Repo], HexRegistry, State)
                       end
                   end,
             handle_missing_no_exception(Fun, Dep, State)

test/mock_pkg_resource.erl

@@ -171,7 +171,7 @@ to_index(AllDeps, Dict, Repos) ->
                              DKB <- [ec_cnv:to_binary(DK)],
                              DVB <- [ec_cnv:to_binary(DV)]],
               Repo = rebar_test_utils:random_element(Repos),
-              ets:insert(?PACKAGE_TABLE, #package{key={N, V, Repo},
+              ets:insert(?PACKAGE_TABLE, #package{key={N, ec_semver:parse(V), Repo},
                                                   dependencies=parse_deps(DepsList),
                                                   retired=false,
                                                   checksum = <<"checksum">>})
@@ -179,11 +179,11 @@ to_index(AllDeps, Dict, Repos) ->
 
     lists:foreach(fun({{Name, Vsn}, _}) ->
                           case lists:any(fun(R) ->
-                                                 ets:member(?PACKAGE_TABLE, {ec_cnv:to_binary(Name), Vsn, R})
+                                                 ets:member(?PACKAGE_TABLE, {ec_cnv:to_binary(Name), ec_semver:parse(Vsn), R})
                                          end, Repos) of
                               false ->
                                   Repo = rebar_test_utils:random_element(Repos),
-                                  ets:insert(?PACKAGE_TABLE, #package{key={ec_cnv:to_binary(Name), Vsn, Repo},
+                                  ets:insert(?PACKAGE_TABLE, #package{key={ec_cnv:to_binary(Name), ec_semver:parse(Vsn), Repo},
                                                                       dependencies=[],
                                                                       retired=false,
                                                                       checksum = <<"checksum">>});

test/rebar_pkg_SUITE.erl

@@ -226,13 +226,13 @@ find_highest_matching(_Config) ->
     State = rebar_state:new(),
     {ok, Vsn} = rebar_packages:find_highest_matching_(
                   <<"goodpkg">>, <<"1.0.0">>, #{name => <<"hexpm">>}, ?PACKAGE_TABLE, State),
-    ?assertEqual(<<"1.0.1">>, Vsn),
+    ?assertEqual({{1,0,1},{[],[]}}, Vsn),
     {ok, Vsn1} = rebar_packages:find_highest_matching(
                    <<"goodpkg">>, <<"1.0">>, #{name => <<"hexpm">>}, ?PACKAGE_TABLE, State),
-    ?assertEqual(<<"1.1.1">>, Vsn1),
+    ?assertEqual({{1,1,1},{[],[]}}, Vsn1),
     {ok, Vsn2} = rebar_packages:find_highest_matching(
                    <<"goodpkg">>, <<"2.0">>, #{name => <<"hexpm">>}, ?PACKAGE_TABLE, State),
-    ?assertEqual(<<"2.0.0">>, Vsn2),
+    ?assertEqual({{2,0,0},{[],[]}}, Vsn2),
 
     %% regression test. ~> constraints higher than the available packages would result
     %% in returning the first package version instead of 'none'.
@@ -265,7 +265,7 @@ mock_config(Name, Config) ->
     lists:foreach(fun({{N, Vsn}, [Deps, Checksum, _]}) ->
                           case ets:member(?PACKAGE_TABLE, {ec_cnv:to_binary(N), Vsn, <<"hexpm">>}) of
                               false ->
-                                  ets:insert(?PACKAGE_TABLE, #package{key={ec_cnv:to_binary(N), Vsn, <<"hexpm">>},
+                                  ets:insert(?PACKAGE_TABLE, #package{key={ec_cnv:to_binary(N), ec_semver:parse(Vsn), <<"hexpm">>},
                                                                       dependencies=Deps,
                                                                       retired=false,
                                                                       checksum=Checksum});

test/rebar_pkg_alias_SUITE.erl

@@ -229,7 +229,7 @@ mock_config(Name, Config) ->
     lists:foreach(fun({{N, Vsn}, [Deps, Checksum, _]}) ->
                           case ets:member(?PACKAGE_TABLE, {ec_cnv:to_binary(N), Vsn, <<"hexpm">>}) of
                               false ->
-                                  ets:insert(?PACKAGE_TABLE, #package{key={ec_cnv:to_binary(N), Vsn, <<"hexpm">>},
+                                  ets:insert(?PACKAGE_TABLE, #package{key={ec_cnv:to_binary(N), ec_semver:parse(Vsn), <<"hexpm">>},
                                                                       dependencies=[{DAppName, {pkg, DN, DV, undefined}} || {DN, DV, _, DAppName} <- Deps],
                                                                       retired=false,
                                                                       checksum=Checksum});

test/rebar_pkg_repos_SUITE.erl

@@ -13,7 +13,7 @@ all() ->
 
 groups() ->
     [{resolve_version, [use_first_repo_match, use_exact_with_hash, fail_repo_update,
-                        ignore_match_in_excluded_repo]}].
+                        ignore_match_in_excluded_repo, optional_prereleases]}].
 
 init_per_group(resolve_version, Config) ->
     Repo1 = <<"test-repo-1">>,
@@ -22,14 +22,17 @@ init_per_group(resolve_version, Config) ->
     Hexpm = <<"hexpm">>,
     Repos = [Repo1, Repo2, Repo3, Hexpm],
 
-    Deps = [{"A", "0.1.1", <<"good checksum">>, Repo1},
-            {"A", "0.1.1", <<"good checksum">>, Repo2},
-            {"B", "1.0.0", Repo1},
-            {"B", "2.0.0", Repo2},
-            {"B", "1.4.0", Repo3},
-            {"B", "1.4.3", Hexpm},
-            {"C", "1.3.1", <<"bad checksum">>, Repo1},
-            {"C", "1.3.1", <<"good checksum">>, Repo2}],
+    Deps = [{"A", "0.1.1", <<"good checksum">>, Repo1, false},
+            {"A", "0.1.1", <<"good checksum">>, Repo2, false},
+            {"B", "1.0.0", Repo1, false},
+            {"B", "2.0.0", Repo2, false},
+            {"B", "1.4.0", Repo3, false},
+            {"B", "1.4.3", Hexpm, false},
+            {"B", "1.4.6", Hexpm, #{reason => 'RETIRED_INVALID'}},
+            {"B", "1.5.0", Hexpm, false},
+            {"B", "1.5.6-rc.0", Hexpm, true},
+            {"C", "1.3.1", <<"bad checksum">>, Repo1, false},
+            {"C", "1.3.1", <<"good checksum">>, Repo2, false}],
     [{deps, Deps}, {repos, Repos} | Config];
 init_per_group(_, Config) ->
     Config.
@@ -96,6 +99,20 @@ init_per_testcase(ignore_match_in_excluded_repo, Config) ->
     %% fail when the first repo is updated since it doesn't have a matching package
     %% should continue anyway
     [_, _, Repo3 | _] = Repos,
+    meck:expect(rebar_packages, update_package,
+                fun(_, _, _State) -> ok  end),
+    meck:expect(rebar_packages, verify_table,
+                fun(_State) -> true end),
+
+    [{state, State} | Config];
+init_per_testcase(optional_prereleases, Config) ->
+    Deps = ?config(deps, Config),
+    Repos = ?config(repos, Config),
+
+    State = setup_deps_and_repos(Deps, Repos),
+
+    meck:new(rebar_packages, [passthrough, no_link]),
+
     meck:expect(rebar_packages, update_package,
                 fun(_, _, _State) -> ok  end),
     meck:expect(rebar_packages, verify_table,
@@ -120,7 +137,8 @@ end_per_testcase(Case, _Config) when Case =:= auth_merging ;
 end_per_testcase(Case, _Config) when Case =:= use_first_repo_match ;
                                      Case =:= use_exact_with_hash ;
                                      Case =:= fail_repo_update ;
-                                     Case =:= ignore_match_in_excluded_repo ->
+                                     Case =:= ignore_match_in_excluded_repo ;
+                                     Case =:= optional_prereleases ->
     meck:unload(rebar_packages);
 end_per_testcase(_, _) ->
     ok.
@@ -250,14 +268,14 @@ organization_merging(_Config) ->
 use_first_repo_match(Config) ->
     State = ?config(state, Config),
 
-    ?assertMatch({ok,{package,{<<"B">>, <<"2.0.0">>, Repo2},
+    ?assertMatch({ok,{package,{<<"B">>, {{2,0,0}, {[],[]}}, Repo2},
                       <<"some checksum">>, false, []},
                   #{name := Repo2,
                     http_adapter_config := #{profile := rebar}}},
                  rebar_packages:resolve_version(<<"B">>, <<"> 1.4.0">>, undefined,
                                                 ?PACKAGE_TABLE, State)),
 
-    ?assertMatch({ok,{package,{<<"B">>, <<"1.4.0">>, Repo3},
+    ?assertMatch({ok,{package,{<<"B">>, {{1,4,0}, {[],[]}}, Repo3},
                       <<"some checksum">>, false, []},
                   #{name := Repo3,
                     http_adapter_config := #{profile := rebar}}},
@@ -268,7 +286,7 @@ use_first_repo_match(Config) ->
 use_exact_with_hash(Config) ->
     State = ?config(state, Config),
 
-    ?assertMatch({ok,{package,{<<"C">>, <<"1.3.1">>, Repo2},
+    ?assertMatch({ok,{package,{<<"C">>, {{1,3,1}, {[],[]}}, Repo2},
                       <<"good checksum">>, false, []},
                   #{name := Repo2,
                     http_adapter_config := #{profile := rebar}}},
@@ -278,7 +296,7 @@ use_exact_with_hash(Config) ->
 fail_repo_update(Config) ->
     State = ?config(state, Config),
 
-    ?assertMatch({ok,{package,{<<"B">>, <<"1.4.0">>, Repo3},
+    ?assertMatch({ok,{package,{<<"B">>, {{1,4,0}, {[],[]}}, Repo3},
                       <<"some checksum">>, false, []},
                   #{name := Repo3,
                     http_adapter_config := #{profile := rebar}}},
@@ -289,24 +307,51 @@ ignore_match_in_excluded_repo(Config) ->
     State = ?config(state, Config),
     Repos = ?config(repos, Config),
 
-    ?assertMatch({ok,{package,{<<"B">>, <<"1.4.3">>, Hexpm},
-                      <<"some checksum">>, false, []},
+    ?assertMatch({ok,{package,{<<"B">>, {{1,4,6}, {[],[]}}, Hexpm},
+                      <<"some checksum">>, #{reason := 'RETIRED_INVALID'}, []},
                   #{name := Hexpm,
                     http_adapter_config := #{profile := rebar}}},
                  rebar_packages:resolve_version(<<"B">>, <<"~> 1.4.0">>, undefined,
                                                 ?PACKAGE_TABLE, State)),
 
     [_, Repo2 | _] = Repos,
-    ?assertMatch({ok,{package,{<<"A">>, <<"0.1.1">>, Repo2},
+    ?assertMatch({ok,{package,{<<"A">>, {{0,1,1}, {[],[]}}, Repo2},
                       <<"good checksum">>, false, []},
                   #{name := Repo2,
                     http_adapter_config := #{profile := rebar}}},
                  rebar_packages:resolve_version(<<"A">>, <<"0.1.1">>, <<"good checksum">>,
                                                 ?PACKAGE_TABLE, State)).
 
+optional_prereleases(Config) ->
+    State = ?config(state, Config),
+
+    ?assertMatch({ok,{package,{<<"B">>, {{1,5,0}, {[],[]}}, Hexpm},
+                      <<"some checksum">>, false, []},
+                  #{name := Hexpm,
+                    http_adapter_config := #{profile := rebar}}},
+                 rebar_packages:resolve_version(<<"B">>, <<"~> 1.5.0">>, undefined,
+                                                ?PACKAGE_TABLE, State)),
+
+    ?assertMatch({ok,{package,{<<"B">>, {{1,5,6}, {[<<"rc">>,0],[]}}, Hexpm},
+                      <<"some checksum">>, true, []},
+                  #{name := Hexpm,
+                    http_adapter_config := #{profile := rebar}}},
+                 rebar_packages:resolve_version(<<"B">>, <<"1.5.6-rc.0">>, <<"some checksum">>,
+                                                ?PACKAGE_TABLE, State)),
+
+    %% allow prerelease through configuration
+    State1 = rebar_state:set(State, deps_allow_prerelease, true),
+    ?assertMatch({ok,{package,{<<"B">>, {{1,5,6}, {[<<"rc">>,0],[]}}, Hexpm},
+                      <<"some checksum">>, true, []},
+                  #{name := Hexpm,
+                    http_adapter_config := #{profile := rebar}}},
+                 rebar_packages:resolve_version(<<"B">>, <<"~> 1.5.0">>, <<"some checksum">>,
+                                                ?PACKAGE_TABLE, State1)).
+
 %%
 
 setup_deps_and_repos(Deps, Repos) ->
+    catch ets:delete(?PACKAGE_TABLE),
     true = rebar_packages:new_package_table(),
     insert_deps(Deps),
     State = rebar_state:new([{hex, [{repos, [#{name => R} || R <- Repos]}]}]),
@@ -314,18 +359,18 @@ setup_deps_and_repos(Deps, Repos) ->
 
 
 insert_deps(Deps) ->
-    lists:foreach(fun({Name, Version, Repo}) ->
+    lists:foreach(fun({Name, Version, Repo, Retired}) ->
                           ets:insert(?PACKAGE_TABLE, #package{key={rebar_utils:to_binary(Name),
-                                                                   rebar_utils:to_binary(Version),
+                                                                   ec_semver:parse(Version),
                                                                    rebar_utils:to_binary(Repo)},
                                                               dependencies=[],
-                                                              retired=false,
+                                                              retired=Retired,
                                                               checksum = <<"some checksum">>});
-                     ({Name, Version, Checksum, Repo}) ->
+                     ({Name, Version, Checksum, Repo, Retired}) ->
                           ets:insert(?PACKAGE_TABLE, #package{key={rebar_utils:to_binary(Name),
-                                                                   rebar_utils:to_binary(Version),
+                                                                   ec_semver:parse(Version),
                                                                    rebar_utils:to_binary(Repo)},
                                                               dependencies=[],
-                                                              retired=false,
+                                                              retired=Retired,
                                                               checksum = Checksum})
                   end, Deps).