Browse Source

replace package management with hex.pm

pull/145/head
Tristan Sloughter 10 years ago
parent
commit
410e2bcdec
14 changed files with 326 additions and 157 deletions
  1. +1
    -1
      Makefile
  2. +3
    -3
      src/rebar_app_info.erl
  3. +2
    -8
      src/rebar_erlc_compiler.erl
  4. +15
    -12
      src/rebar_fetch.erl
  5. +9
    -9
      src/rebar_git_resource.erl
  6. +13
    -5
      src/rebar_packages.erl
  7. +5
    -3
      src/rebar_pkg_resource.erl
  8. +24
    -23
      src/rebar_prv_install_deps.erl
  9. +49
    -15
      src/rebar_prv_update.erl
  10. +2
    -2
      src/rebar_resource.erl
  11. +1
    -1
      test/mock_git_resource.erl
  12. +11
    -8
      test/mock_pkg_resource.erl
  13. +187
    -62
      test/rebar_deps_SUITE.erl
  14. +4
    -5
      test/rebar_upgrade_SUITE.erl

+ 1
- 1
Makefile View File

@ -6,7 +6,7 @@ RETEST=$(PWD)/deps/retest/retest
DEPS_PLT=$(CURDIR)/.depsolver_plt
all:
./bootstrap/bootstrap
@rebar get-deps compile escriptize
clean:
@rm -rf rebar3 ebin/*.beam inttest/rt.work rt.work .eunit

+ 3
- 3
src/rebar_app_info.erl View File

@ -79,7 +79,7 @@ new(AppName, Vsn) ->
new(AppName, Vsn, Dir) ->
{ok, #app_info_t{name=ec_cnv:to_binary(AppName),
original_vsn=Vsn,
dir=Dir}}.
dir=ec_cnv:to_list(Dir)}}.
%% @doc build a complete version of the app info with all fields set.
-spec new(atom() | binary() | string(), binary() | string(), file:name(), list()) ->
@ -87,7 +87,7 @@ new(AppName, Vsn, Dir) ->
new(AppName, Vsn, Dir, Deps) ->
{ok, #app_info_t{name=ec_cnv:to_binary(AppName),
original_vsn=Vsn,
dir=Dir,
dir=ec_cnv:to_list(Dir),
deps=Deps}}.
%% @doc discover a complete version of the app info with all fields set.
@ -200,7 +200,7 @@ dir(#app_info_t{dir=Dir}) ->
-spec dir(t(), file:name()) -> t().
dir(AppInfo=#app_info_t{}, Dir) ->
AppInfo#app_info_t{dir=Dir}.
AppInfo#app_info_t{dir=ec_cnv:to_list(Dir)}.
-spec ebin_dir(t()) -> file:name().
ebin_dir(#app_info_t{dir=Dir}) ->

+ 2
- 8
src/rebar_erlc_compiler.erl View File

@ -144,17 +144,13 @@ doterl_compile(Config, Dir, MoreSources, ErlOpts) ->
filename:join(Dir, X)
end, rebar_dir:src_dirs(proplists:append_values(src_dirs, ErlOpts))),
AllErlFiles = gather_src(SrcDirs, []) ++ MoreSources,
%% NOTE: If and when erl_first_files is not inherited anymore
%% (rebar_state:get instead of rebar_state:get_list), consider
%% logging a warning message for any file listed in erl_first_files which
%% wasn't found via gather_src.
%% Issue: rebar/rebar3#140 (fix matching based on same path + order of
%% erl_first_files)
ErlFirstFiles = get_erl_first_files(ErlFirstFilesConf, AllErlFiles),
RestErls = [ File || File <- AllErlFiles,
not lists:member(File, ErlFirstFiles) ],
%% Make sure that ebin/ exists and is on the path
ok = filelib:ensure_dir(filename:join([Dir, "ebin", "dummy.beam"])),
CurrPath = code:get_path(),
@ -634,5 +630,3 @@ log_files(Prefix, Files) ->
_ ->
?DEBUG("~s:~n~p", [Prefix, Files])
end.

+ 15
- 12
src/rebar_fetch.erl View File

@ -8,12 +8,13 @@
-module(rebar_fetch).
-export([lock_source/2,
download_source/2,
download_source/3,
needs_update/2]).
-export([format_error/1]).
-include("rebar.hrl").
-include_lib("providers/include/providers.hrl").
%% map short versions of resources to module names
-define(RESOURCES, [{git, rebar_git_resource}, {pkg, rebar_pkg_resource}]).
@ -24,13 +25,14 @@ lock_source(AppDir, Source) ->
Module = get_resource_type(Source),
Module:lock(AppDir, Source).
-spec download_source(file:filename_all(), rebar_resource:resource()) -> true | {error, any()}.
download_source(AppDir, Source) ->
-spec download_source(file:filename_all(), rebar_resource:resource(), rebar_state:t()) ->
true | {error, any()}.
download_source(AppDir, Source, State) ->
try
Module = get_resource_type(Source),
TmpDir = ec_file:insecure_mkdtemp(),
AppDir1 = ec_cnv:to_list(AppDir),
case Module:download(TmpDir, Source) of
case Module:download(TmpDir, Source, State) of
{ok, _} ->
ec_file:mkdir_p(AppDir1),
code:del_path(filename:absname(filename:join(AppDir1, "ebin"))),
@ -38,20 +40,21 @@ download_source(AppDir, Source) ->
ok = ec_file:copy(TmpDir, filename:absname(AppDir1), [recursive]),
true;
{tarball, File} ->
Contents = filename:join(TmpDir, "contents"),
ec_file:mkdir_p(AppDir1),
ok = erl_tar:extract(File, [{cwd, TmpDir}
,compressed]),
BaseName = filename:basename(AppDir1),
[FromDir] = filelib:wildcard(filename:join(TmpDir, BaseName++"-*")),
ec_file:mkdir_p(Contents),
ok = erl_tar:extract(File, [{cwd, TmpDir}]),
ok = erl_tar:extract(filename:join(TmpDir, "contents.tar.gz"),
[{cwd, Contents}, compressed]),
code:del_path(filename:absname(filename:join(AppDir1, "ebin"))),
ec_file:remove(filename:absname(AppDir1), [recursive]),
ok = ec_file:copy(FromDir, filename:absname(AppDir1), [recursive]),
true = code:add_patha(filename:join(AppDir1, "ebin")),
ok = ec_file:copy(Contents, filename:absname(AppDir1), [recursive]),
true
end
catch
_:_ ->
{error, {rebar_fetch, {fetch_fail, Source}}}
C:T ->
?DEBUG("rebar_fetch exception ~p ~p ~p", [C, T, erlang:get_stacktrace()]),
throw(?PRV_ERROR({fetch_fail, Source}))
end.
-spec needs_update(file:filename_all(), rebar_resource:resource()) -> boolean() | {error, string()}.

+ 9
- 9
src/rebar_git_resource.erl View File

@ -5,7 +5,7 @@
-behaviour(rebar_resource).
-export([lock/2
,download/2
,download/3
,needs_update/2
,make_vsn/1]).
@ -75,28 +75,28 @@ parse_git_url("https://" ++ HostPath) ->
[Host | Path] = string:tokens(HostPath, "/"),
{Host, filename:rootname(filename:join(Path), ".git")}.
download(Dir, {git, Url}) ->
download(Dir, {git, Url}, State) ->
?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []),
download(Dir, {git, Url, {branch, "master"}});
download(Dir, {git, Url, ""}) ->
download(Dir, {git, Url, {branch, "master"}}, State);
download(Dir, {git, Url, ""}, State) ->
?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []),
download(Dir, {git, Url, {branch, "master"}});
download(Dir, {git, Url, {branch, Branch}}) ->
download(Dir, {git, Url, {branch, "master"}}, State);
download(Dir, {git, Url, {branch, Branch}}, _State) ->
ok = filelib:ensure_dir(Dir),
rebar_utils:sh(?FMT("git clone ~s ~s -b ~s --single-branch",
[Url, filename:basename(Dir), Branch]),
[{cd, filename:dirname(Dir)}]);
download(Dir, {git, Url, {tag, Tag}}) ->
download(Dir, {git, Url, {tag, Tag}}, _State) ->
ok = filelib:ensure_dir(Dir),
rebar_utils:sh(?FMT("git clone ~s ~s -b ~s --single-branch",
[Url, filename:basename(Dir), Tag]),
[{cd, filename:dirname(Dir)}]);
download(Dir, {git, Url, {ref, Ref}}) ->
download(Dir, {git, Url, {ref, Ref}}, _State) ->
ok = filelib:ensure_dir(Dir),
rebar_utils:sh(?FMT("git clone -n ~s ~s", [Url, filename:basename(Dir)]),
[{cd, filename:dirname(Dir)}]),
rebar_utils:sh(?FMT("git checkout -q ~s", [Ref]), [{cd, Dir}]);
download(Dir, {git, Url, Rev}) ->
download(Dir, {git, Url, Rev}, _State) ->
?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []),
ok = filelib:ensure_dir(Dir),
rebar_utils:sh(?FMT("git clone -n ~s ~s", [Url, filename:basename(Dir)]),

+ 13
- 5
src/rebar_packages.erl View File

@ -13,13 +13,21 @@
-spec get_packages(rebar_state:t()) -> {rebar_dict(), rebar_digraph()}.
get_packages(State) ->
RebarDir = rebar_dir:global_config_dir(State),
PackagesFile = filename:join(RebarDir, "packages"),
case ec_file:exists(PackagesFile) of
RegistryDir = filename:join(RebarDir, "packages"),
DictFile = filename:join(RegistryDir, "dict"),
Edges = filename:join(RegistryDir, "edges"),
Vertices = filename:join(RegistryDir, "vertices"),
Neighbors = filename:join(RegistryDir, "neighbors"),
case lists:all(fun(X) -> filelib:is_file(X) end, [DictFile, Edges, Vertices, Neighbors]) of
true ->
try
{ok, Binary} = file:read_file(PackagesFile),
{Dict, Graph} = binary_to_term(Binary),
{Dict, rebar_digraph:restore_graph(Graph)}
{ok, DictBinary} = file:read_file(DictFile),
Dict = binary_to_term(DictBinary),
{ok, EdgesTab} = ets:file2tab(Edges),
{ok, VerticesTab} = ets:file2tab(Vertices),
{ok, NeighborsTab} = ets:file2tab(Neighbors),
{Dict, {digraph, EdgesTab, VerticesTab, NeighborsTab, true}}
catch
_:_ ->
?ERROR("Bad packages index, try to fix with `rebar3 update`", []),

+ 5
- 3
src/rebar_pkg_resource.erl View File

@ -5,7 +5,7 @@
-behaviour(rebar_resource).
-export([lock/2
,download/2
,download/3
,needs_update/2
,make_vsn/1]).
@ -23,9 +23,11 @@ needs_update(Dir, {pkg, _Name, Vsn, _Url}) ->
true
end.
download(Dir, {pkg, _Name, _Vsn, Url}) ->
download(Dir, {pkg, Name, Vsn}, State) ->
TmpFile = filename:join(Dir, "package.tar.gz"),
{ok, saved_to_file} = httpc:request(get, {binary_to_list(Url), []}, [], [{stream, TmpFile}]),
CDN = rebar_state:get(State, rebar_packages_cdn, "https://s3.amazonaws.com/s3.hex.pm/tarballs"),
Url = string:join([CDN, binary_to_list(<<Name/binary, "-", Vsn/binary, ".tar">>)], "/"),
{ok, saved_to_file} = httpc:request(get, {Url, []}, [], [{stream, TmpFile}]),
{tarball, TmpFile}.
make_vsn(_) ->

+ 24
- 23
src/rebar_prv_install_deps.erl View File

@ -75,19 +75,19 @@ do(State) ->
Profiles = rebar_state:current_profiles(State),
ProjectApps = rebar_state:project_apps(State),
{SrcApps, State1} =
lists:foldl(fun(Profile, {SrcAppsAcc, StateAcc}) ->
{Apps, State1} =
lists:foldl(fun(Profile, {AppsAcc, StateAcc}) ->
Locks = rebar_state:get(StateAcc, {locks, Profile}, []),
{ok, NewSrcApps, NewState} =
{ok, NewApps, NewState} =
handle_deps(Profile
,StateAcc
,rebar_state:get(StateAcc, {deps, Profile}, [])
,Locks),
{NewSrcApps++SrcAppsAcc, NewState}
{NewApps++AppsAcc, NewState}
end, {[], State}, lists:reverse(Profiles)),
Source = ProjectApps ++ SrcApps,
case find_cycles(Source ++ rebar_state:all_deps(State1)) of
Source = ProjectApps ++ Apps,
case find_cycles(Source) of
{cycles, Cycles} ->
?PRV_ERROR({cycles, Cycles});
{error, Error} ->
@ -117,7 +117,9 @@ find_cycles(Apps) ->
-spec format_error(any()) -> iolist().
format_error({parse_dep, Dep}) ->
io_lib:format("Failed parsing dep ~p~n", [Dep]);
io_lib:format("Failed parsing dep ~p", [Dep]);
format_error({missing_package, Package, Version}) ->
io_lib:format("Package not found in registry: ~s-~s", [Package, Version]);
format_error({cycles, Cycles}) ->
Prints = [["applications: ",
[io_lib:format("~s ", [Dep]) || Dep <- Cycle],
@ -178,7 +180,7 @@ handle_deps(Profile, State, Deps, Upgrade, Locks) ->
%% Sort all apps to build order
State3 = rebar_state:all_deps(State2, AllDeps),
{ok, SrcApps, State3}.
{ok, AllDeps, State3}.
%% ===================================================================
%% Internal functions
@ -193,7 +195,7 @@ update_pkg_deps(Profile, Pkgs, Packages, Upgrade, Seen, State) ->
,Packages
,Pkg),
{SeenAcc1, StateAcc1} = maybe_lock(Profile, AppInfo, SeenAcc, StateAcc, 0),
case maybe_fetch(AppInfo, Upgrade, SeenAcc1) of
case maybe_fetch(AppInfo, Upgrade, SeenAcc1, State) of
true ->
{[AppInfo | Acc], SeenAcc1, StateAcc1};
false ->
@ -227,15 +229,14 @@ maybe_lock(Profile, AppInfo, Seen, State, Level) ->
package_to_app(DepsDir, Packages, {Name, Vsn}) ->
case dict:find({Name, Vsn}, Packages) of
error ->
{error, missing_package};
throw(?PRV_ERROR({missing_package, Name, Vsn}));
{ok, P} ->
PkgDeps = [{PkgName, PkgVsn}
|| {PkgName,PkgVsn} <- proplists:get_value(<<"deps">>, P, [])],
Link = proplists:get_value(<<"link">>, P, ""),
{ok, AppInfo} = rebar_app_info:new(Name, Vsn),
AppInfo1 = rebar_app_info:deps(AppInfo, PkgDeps),
AppInfo2 = rebar_app_info:dir(AppInfo1, rebar_dir:deps_dir(DepsDir, Name)),
rebar_app_info:source(AppInfo2, {pkg, Name, Vsn, Link})
rebar_app_info:source(AppInfo2, {pkg, Name, Vsn})
end.
-spec update_src_deps(atom(), non_neg_integer(), list(), list(), list(), rebar_state:t(), boolean(), sets:set(binary()), list()) -> {rebar_state:t(), list(), list(), sets:set(binary())}.
@ -282,7 +283,7 @@ update_src_deps(Profile, Level, SrcDeps, PkgDeps, SrcApps, State, Upgrade, Seen,
,StateAcc1
,LocksAcc);
_ ->
maybe_fetch(AppInfo, false, SeenAcc),
maybe_fetch(AppInfo, false, SeenAcc, StateAcc),
handle_dep(AppInfo
,SrcDepsAcc
,PkgDepsAcc
@ -306,7 +307,7 @@ handle_upgrade(AppInfo, SrcDeps, PkgDeps, SrcApps, Level, State, Locks) ->
Name = rebar_app_info:name(AppInfo),
case lists:keyfind(Name, 1, Locks) of
false ->
case maybe_fetch(AppInfo, true, sets:new()) of
case maybe_fetch(AppInfo, true, sets:new(), State) of
true ->
handle_dep(AppInfo
,SrcDeps
@ -357,8 +358,8 @@ handle_dep(State, DepsDir, AppInfo, Locks, Level) ->
{AppInfo2, SrcDeps, PkgDeps, Locks++NewLocks}.
-spec maybe_fetch(rebar_app_info:t(), boolean() | {true, binary(), integer()},
sets:set(binary())) -> boolean().
maybe_fetch(AppInfo, Upgrade, Seen) ->
sets:set(binary()), rebar_state:t()) -> boolean().
maybe_fetch(AppInfo, Upgrade, Seen, State) ->
AppDir = ec_cnv:to_list(rebar_app_info:dir(AppInfo)),
Apps = rebar_app_discover:find_apps(["_checkouts"], all),
case rebar_app_utils:find(rebar_app_info:name(AppInfo), Apps) of
@ -368,13 +369,13 @@ maybe_fetch(AppInfo, Upgrade, Seen) ->
error ->
case not app_exists(AppDir) of
true ->
fetch_app(AppInfo, AppDir);
fetch_app(AppInfo, AppDir, State);
false ->
case sets:is_element(rebar_app_info:name(AppInfo), Seen) of
true ->
false;
false ->
maybe_upgrade(AppInfo, AppDir, Upgrade)
maybe_upgrade(AppInfo, AppDir, Upgrade, State)
end
end
end.
@ -457,25 +458,25 @@ app_exists(AppDir) ->
end
end.
fetch_app(AppInfo, AppDir) ->
fetch_app(AppInfo, AppDir, State) ->
?INFO("Fetching ~s (~p)", [rebar_app_info:name(AppInfo), rebar_app_info:source(AppInfo)]),
Source = rebar_app_info:source(AppInfo),
case rebar_fetch:download_source(AppDir, Source) of
case rebar_fetch:download_source(AppDir, Source, State) of
{error, Reason} ->
throw(Reason);
Result ->
Result
end.
maybe_upgrade(AppInfo, AppDir, false) ->
maybe_upgrade(AppInfo, AppDir, false, _State) ->
Source = rebar_app_info:source(AppInfo),
rebar_fetch:needs_update(AppDir, Source);
maybe_upgrade(AppInfo, AppDir, true) ->
maybe_upgrade(AppInfo, AppDir, true, State) ->
Source = rebar_app_info:source(AppInfo),
case rebar_fetch:needs_update(AppDir, Source) of
true ->
?INFO("Updating ~s", [rebar_app_info:name(AppInfo)]),
case rebar_fetch:download_source(AppDir, Source) of
case rebar_fetch:download_source(AppDir, Source, State) of
{error, Reason} ->
throw(Reason);
Result ->

+ 49
- 15
src/rebar_prv_update.erl View File

@ -34,18 +34,22 @@ init(State) ->
do(State) ->
?INFO("Updating package index...", []),
try
Url = url(State),
Url = rebar_state:get(State, rebar_packages_cdn, "https://s3.amazonaws.com/s3.hex.pm/registry.ets.gz"),
TmpDir = ec_file:insecure_mkdtemp(),
TmpFile = filename:join(TmpDir, "packages"),
Home = rebar_dir:home_dir(),
PackagesFile = filename:join([Home, ?CONFIG_DIR, "packages"]),
filelib:ensure_dir(PackagesFile),
{ok, _RequestId} = httpc:request(get, {Url, [{"Accept", "application/erlang"}]},
[], [{stream, TmpFile}, {sync, true}]),
ok = ec_file:copy(TmpFile, PackagesFile)
TmpFile = filename:join(TmpDir, "packages.gz"),
HexFile = filename:join(TmpDir, "packages"),
{ok, _RequestId} = httpc:request(get, {Url, []},
[], [{stream, TmpFile}, {sync, true}]),
{ok, Data} = file:read_file(TmpFile),
Unzipped = zlib:gunzip(Data),
ok = file:write_file(HexFile, Unzipped),
{Dict, Graph} = hex_to_graph(HexFile),
write_registry(Dict, Graph, State),
ok
catch
_:_ ->
{error, {?MODULE, package_index_write}}
E:C ->
io:format("E C ~p ~p~n", [E, C]),
throw({error, {?MODULE, package_index_write}})
end,
{ok, State}.
@ -54,9 +58,39 @@ do(State) ->
format_error(package_index_write) ->
"Failed to write package index.".
url(State) ->
ErtsVsn = erlang:system_info(version),
write_registry(Dict, {digraph, Edges, Vertices, Neighbors, _}, State) ->
Dir = rebar_dir:global_config_dir(State),
RegistryDir = filename:join(Dir, "packages"),
filelib:ensure_dir(filename:join(RegistryDir, "dummy")),
ets:tab2file(Edges, filename:join(RegistryDir, "edges")),
ets:tab2file(Vertices, filename:join(RegistryDir, "vertices")),
ets:tab2file(Neighbors, filename:join(RegistryDir, "neighbors")),
file:write_file(filename:join(RegistryDir, "dict"), term_to_binary(Dict)).
Qs = [io_lib:format("~s=~s", [X, Y]) || {X, Y} <- [{"erts", ErtsVsn}]],
Url = rebar_state:get(State, rebar_packages_url, "http://packages.rebar3.org/packages"),
Url ++ "?" ++ string:join(Qs, "&").
hex_to_graph(Filename) ->
{ok, T} = ets:file2tab(Filename),
Graph = digraph:new(),
ets:foldl(fun({Pkg, [Versions]}, ok) when is_binary(Pkg), is_list(Versions) ->
lists:foreach(fun(Version) ->
digraph:add_vertex(Graph, {Pkg, Version}, 1)
end, Versions);
(_, ok) ->
ok
end, ok, T),
Dict1 = ets:foldl(fun({{Pkg, PkgVsn}, [Deps | _]}, Dict) ->
DepsList = lists:foldl(fun([Dep, DepVsn, _, _], DepsListAcc) ->
case DepVsn of
<<"~> ", Vsn/binary>> ->
digraph:add_edge(Graph, {Pkg, PkgVsn}, {Dep, Vsn}),
[{Dep, DepVsn} | DepsListAcc];
Vsn ->
digraph:add_edge(Graph, {Pkg, PkgVsn}, {Dep, Vsn}),
[{Dep, Vsn} | DepsListAcc]
end
end, [], Deps),
dict:store({Pkg, PkgVsn}, DepsList, Dict);
(_, Dict) ->
Dict
end, dict:new(), T),
{Dict1, Graph}.

+ 2
- 2
src/rebar_resource.erl View File

@ -23,7 +23,7 @@
-spec behaviour_info(atom()) -> [{atom(), arity()}] | undefined.
behaviour_info(callbacks) ->
[{lock, 2},
{download, 2},
{download, 3},
{needs_update,2},
{make_vsn, 1}];
behaviour_info(_) ->
@ -33,7 +33,7 @@ behaviour_info(_) ->
-callback lock(file:filename_all(), tuple()) ->
rebar_resource:resource().
-callback download(file:filename_all(), tuple()) ->
-callback download(file:filename_all(), tuple(), rebar_state:t()) ->
{tarball, file:filename_all()} | {ok, any()} | {error, any()}.
-callback needs_update(file:filename_all(), tuple()) ->
boolean().

+ 1
- 1
test/mock_git_resource.erl View File

@ -103,7 +103,7 @@ mock_download(Opts) ->
Overrides = proplists:get_value(override_vsn, Opts, []),
meck:expect(
?MOD, download,
fun (Dir, Git) ->
fun (Dir, Git, _) ->
filelib:ensure_dir(Dir),
{git, Url, {_, Vsn}} = normalize_git(Git, Overrides, Default),
App = app(Url),

+ 11
- 8
test/mock_pkg_resource.erl View File

@ -21,7 +21,7 @@ mock() -> mock([]).
| {not_in_index, [{App, Vsn}]}
| {pkgdeps, [{{App,Vsn}, [Dep]}]},
App :: string(),
Dep :: {App, string(), {pkg, App, Vsn, Url::string()}},
Dep :: {App, string(), {pkg, App, Vsn}},
Vsn :: string().
mock(Opts) ->
meck:new(?MOD, [no_link]),
@ -50,7 +50,7 @@ mock_update(Opts) ->
ToUpdate = proplists:get_value(update, Opts, []),
meck:expect(
?MOD, needs_update,
fun(_Dir, {pkg, App, _Vsn, _Url}) ->
fun(_Dir, {pkg, App, _Vsn}) ->
lists:member(App, ToUpdate)
end).
@ -65,7 +65,7 @@ mock_vsn(_Opts) ->
%% @doc For each app to download, create a dummy app on disk instead.
%% The configuration for this one (passed in from `mock/1') includes:
%%
%% - Specify a version with `{pkg, _, Vsn, _}'
%% - Specify a version with `{pkg, _, Vsn}'
%% - Dependencies for each application must be passed of the form:
%% `{pkgdeps, [{"app1", [{app2, ".*", {pkg, ...}}]}]}' -- basically
%% the `pkgdeps' option takes a key/value list of terms to output directly
@ -74,7 +74,7 @@ mock_download(Opts) ->
Deps = proplists:get_value(pkgdeps, Opts, []),
meck:expect(
?MOD, download,
fun (Dir, {pkg, AppBin, Vsn, _Url}) ->
fun (Dir, {pkg, AppBin, Vsn}, _) ->
App = binary_to_list(AppBin),
filelib:ensure_dir(Dir),
AppDeps = proplists:get_value({App,Vsn}, Deps, []),
@ -83,11 +83,15 @@ mock_download(Opts) ->
[element(1,D) || D <- AppDeps]
),
rebar_test_utils:create_config(Dir, [{deps, AppDeps}]),
Tarball = filename:join([Dir, "package.tar.gz"]),
Tarball = filename:join([Dir, App++"-"++binary_to_list(Vsn)++".tar"]),
Contents = filename:join([Dir, "contents.tar.gz"]),
Files = all_files(rebar_app_info:dir(AppInfo)),
ok = erl_tar:create(Tarball,
ok = erl_tar:create(Contents,
archive_names(Dir, App, Vsn, Files),
[compressed]),
ok = erl_tar:create(Tarball,
[{"contents.tar.gz", Contents}],
[]),
[file:delete(F) || F <- Files],
{tarball, Tarball}
end).
@ -117,8 +121,7 @@ all_files(Dir) ->
filelib:wildcard(filename:join([Dir, "**"])).
archive_names(Dir, App, Vsn, Files) ->
ArchName = App++"-"++binary_to_list(Vsn),
[{ArchName ++ (F -- Dir), F} || F <- Files].
[{(F -- Dir) -- "/", F} || F <- Files].
find_parts(Apps, Skip) -> find_parts(Apps, Skip, dict:new()).

+ 187
- 62
test/rebar_deps_SUITE.erl View File

@ -1,14 +1,16 @@
%% TODO: add tests for:
%% - only part of deps fetched
%% - only part of deps locked
%% - output only shown once
%% - modification asterisk on locked file
-module(rebar_deps_SUITE).
-compile(export_all).
-include_lib("common_test/include/ct.hrl").
-include_lib("eunit/include/eunit.hrl").
all() -> [default_nodep, default_lock].
all() -> [{group, git}, {group, pkg}].
groups() ->
[{all, [], [flat, pick_highest_left, pick_highest_right,
pick_smallest1, pick_smallest2,
circular1, circular2, circular_skip]},
{git, [], [{group, all}]},
{pkg, [], [{group, all}]}].
init_per_suite(Config) ->
application:start(meck),
@ -17,79 +19,202 @@ init_per_suite(Config) ->
end_per_suite(_Config) ->
application:stop(meck).
init_per_group(git, Config) ->
[{deps_type, git} | Config];
init_per_group(pkg, Config) ->
[{deps_type, pkg} | Config];
init_per_group(_, Config) ->
Config.
end_per_group(_, Config) ->
Config.
init_per_testcase(Case, Config) ->
meck:new(io, [no_link, passthrough, unstick]),
setup_project(Case, Config).
{Deps, Warnings, Expect} = deps(Case),
Expected = case Expect of
{ok, List} -> {ok, format_expected_deps(List)};
{error, Reason} -> {error, Reason}
end,
DepsType = ?config(deps_type, Config),
mock_warnings(),
[{expect, Expected},
{warnings, Warnings}
| setup_project(Case, Config, expand_deps(DepsType, Deps))].
end_per_testcase(_, Config) ->
meck:unload(),
Config.
config_content() ->
[{deps, [
{src_a, ".*", {git, "https://example.org/ferd/src_a.git", {branch, "master"}}},
{src_b, {git, "https://example.org/ferd/src_b.git", {branch, "master"}}},
{pkg_a, "1.0.0"}
]},
{profiles,
[{test,
[{deps, [
{tdep, {git, "git://example.org/ferd/tdep.git", {tag, "0.8.2"}}}
]}]
}]}
].
setup_project(Case, Config0) ->
format_expected_deps(Deps) ->
[case Dep of
{N,V} -> {dep, N, V};
N -> {dep, N}
end || Dep <- Deps].
%% format:
%% {Spec,
%% [Warning],
%% {ok, Result} | {error, Reason}}
%%
%% Spec is a list of levelled dependencies of two possible forms:
%% - {"Name", Spec}
%% - {"Name", "Vsn", Spec}
%%
%% Warnings are going to match on mocked ?WARN(...)
%% calls to be evaluated. An empty list means we do not care about
%% warnings, not that no warnings will be printed. This means
%% the list of warning isn't interpreted to be exhaustive, and more
%% warnings may be generated than are listed.
deps(flat) ->
{[{"B", []},
{"C", []}],
[],
{ok, ["B", "C"]}};
deps(pick_highest_left) ->
{[{"B", [{"C", "2", []}]},
{"C", "1", []}],
[{"C","2"}],
{ok, ["B", {"C", "1"}]}};
deps(pick_highest_right) ->
{[{"B", "1", []},
{"C", [{"B", "2", []}]}],
[{"B","2"}],
{ok, [{"B","1"}, "C"]}};
deps(pick_smallest1) ->
{[{"B", [{"D", "1", []}]},
{"C", [{"D", "2", []}]}],
[{"D","2"}],
%% we pick D1 because B < C
{ok, ["B","C",{"D","1"}]}};
deps(pick_smallest2) ->
{[{"C", [{"D", "2", []}]},
{"B", [{"D", "1", []}]}],
[{"D","2"}],
%% we pick D1 because B < C
{ok, ["B","C",{"D","1"}]}};
deps(circular1) ->
{[{"B", [{"A", []}]}, % A is the top-level app
{"C", []}],
[],
{error, {rebar_prv_install_deps, {cycles, [[<<"A">>,<<"B">>]]}}}};
deps(circular2) ->
{[{"B", [{"C", [{"B", []}]}]},
{"C", []}],
[],
{error, {rebar_prv_install_deps, {cycles, [[<<"B">>,<<"C">>]]}}}};
deps(circular_skip) ->
%% Never spot the circular dep due to being to low in the deps tree
%% in source deps
{[{"B", [{"C", "2", [{"B", []}]}]},
{"C", "1", [{"D",[]}]}],
[{"C","2"}],
{ok, ["B", {"C","1"}, "D"]}}.
expand_deps(_, []) -> [];
expand_deps(git, [{Name, Deps} | Rest]) ->
Dep = {Name, ".*", {git, "https://example.org/user/"++Name++".git", "master"}},
[{Dep, expand_deps(git, Deps)} | expand_deps(git, Rest)];
expand_deps(git, [{Name, Vsn, Deps} | Rest]) ->
Dep = {Name, Vsn, {git, "https://example.org/user/"++Name++".git", {tag, Vsn}}},
[{Dep, expand_deps(git, Deps)} | expand_deps(git, Rest)];
expand_deps(pkg, [{Name, Deps} | Rest]) ->
Dep = {pkg, Name, "0.0.0"},
[{Dep, expand_deps(pkg, Deps)} | expand_deps(pkg, Rest)];
expand_deps(pkg, [{Name, Vsn, Deps} | Rest]) ->
Dep = {pkg, Name, Vsn},
[{Dep, expand_deps(pkg, Deps)} | expand_deps(pkg, Rest)].
setup_project(Case, Config0, Deps) ->
DepsType = ?config(deps_type, Config0),
Config = rebar_test_utils:init_rebar_state(
Config0,
atom_to_list(Case)++"_"
atom_to_list(Case)++"_"++atom_to_list(DepsType)++"_"
),
AppDir = ?config(apps, Config),
rebar_test_utils:create_app(AppDir, "A", "0.0.0", [kernel, stdlib]),
TopDeps = proplists:get_value(deps, config_content()),
StringDeps = [erlang:setelement(1, Dep, atom_to_list(element(1,Dep)))
|| Dep <- TopDeps],
TopDeps = top_level_deps(Deps),
RebarConf = rebar_test_utils:create_config(AppDir, [{deps, TopDeps}]),
mock_git_resource:mock([{deps, lists:filter(fun src_dep/1, StringDeps)}]),
mock_pkg_resource:mock([{pkgdeps,
[{{ec_cnv:to_binary(N),
ec_cnv:to_binary(V)},[]}
|| {N,V} <- lists:filter(fun pkg_dep/1, StringDeps)]}]),
case DepsType of
git ->
mock_git_resource:mock([{deps, flat_deps(Deps)}]);
pkg ->
mock_pkg_resource:mock([{pkgdeps, flat_pkgdeps(Deps)}])
end,
[{rebarconfig, RebarConf} | Config].
src_dep(Dep) ->
case element(1, Dep) of
"src_"++_ -> true;
_ -> false
end.
pkg_dep(Dep) ->
case element(1, Dep) of
"pkg_"++_ -> true;
_ -> false
end.
flat_deps([]) -> [];
flat_deps([{{Name,_Vsn,Ref}, Deps} | Rest]) ->
[{{Name,vsn_from_ref(Ref)}, top_level_deps(Deps)}]
++
flat_deps(Deps)
++
flat_deps(Rest).
default_nodep(Config) ->
{ok, RebarConfig} = file:consult(?config(rebarconfig, Config)),
rebar_test_utils:run_and_check(
Config, RebarConfig, ["deps"], {ok, []}
),
History = meck:history(io),
Strings = [io_lib:format(Str, Args) || {_, {io, format, [Str, Args]}, _} <- History],
{match, _} = re:run(Strings, "src_a\\* \\(git source\\)"),
{match, _} = re:run(Strings, "src_b\\* \\(git source\\)"),
{match, _} = re:run(Strings, "pkg_a\\* \\(package 1.0.0\\)").
vsn_from_ref({git, _, {_, Vsn}}) -> Vsn;
vsn_from_ref({git, _, Vsn}) -> Vsn.
default_lock(Config) ->
flat_pkgdeps([]) -> [];
flat_pkgdeps([{{pkg, Name, Vsn}, Deps} | Rest]) ->
[{{iolist_to_binary(Name),iolist_to_binary(Vsn)}, top_level_deps(Deps)}]
++
flat_pkgdeps(Deps)
++
flat_pkgdeps(Rest).
top_level_deps([]) -> [];
top_level_deps([{{pkg, Name, Vsn}, _} | Deps]) ->
[{list_to_atom(Name), Vsn} | top_level_deps(Deps)];
top_level_deps([{{Name, Vsn, Ref}, _} | Deps]) ->
[{list_to_atom(Name), Vsn, Ref} | top_level_deps(Deps)].
app_vsn([]) -> [];
app_vsn([{Source, Deps} | Rest]) ->
{Name, Vsn} = case Source of
{pkg, N, V} -> {N,V};
{N,V,_Ref} -> {N,V}
end,
[{Name, Vsn}] ++ app_vsn(Deps) ++ app_vsn(Rest).
mock_warnings() ->
%% just let it do its thing, we check warnings through
%% the call log.
meck:new(rebar_log, [no_link, passthrough]).
%%% TESTS %%%
flat(Config) -> run(Config).
pick_highest_left(Config) -> run(Config).
pick_highest_right(Config) -> run(Config).
pick_smallest1(Config) -> run(Config).
pick_smallest2(Config) -> run(Config).
circular1(Config) -> run(Config).
circular2(Config) -> run(Config).
circular_skip(Config) -> run(Config).
run(Config) ->
{ok, RebarConfig} = file:consult(?config(rebarconfig, Config)),
rebar_test_utils:run_and_check(
Config, RebarConfig, ["lock"], {ok, []}
Config, RebarConfig, ["install_deps"], ?config(expect, Config)
),
rebar_test_utils:run_and_check(
Config, RebarConfig, ["deps"], {ok, []}
),
History = meck:history(io),
Strings = [io_lib:format(Str, Args) || {_, {io, format, [Str, Args]}, _} <- History],
{match, _} = re:run(Strings, "src_a\\ \\(locked git source\\)"),
{match, _} = re:run(Strings, "src_b\\ \\(locked git source\\)"),
{match, _} = re:run(Strings, "pkg_a\\ \\(locked package 1.0.0\\)").
check_warnings(warning_calls(), ?config(warnings, Config), ?config(deps_type, Config)).
warning_calls() ->
History = meck:history(rebar_log),
[{Str, Args} || {_, {rebar_log, log, [warn, Str, Args]}, _} <- History].
check_warnings(_, [], _) ->
ok;
check_warnings(Warns, [{Name, Vsn} | Rest], Type) ->
ct:pal("Checking for warning ~p in ~p", [{Name,Vsn},Warns]),
?assert(in_warnings(Type, Warns, Name, Vsn)),
check_warnings(Warns, Rest, Type).
in_warnings(git, Warns, NameRaw, VsnRaw) ->
Name = iolist_to_binary(NameRaw),
1 =< length([1 || {_, [AppName, {git, _, {_, Vsn}}]} <- Warns,
AppName =:= Name, Vsn =:= VsnRaw]);
in_warnings(pkg, Warns, NameRaw, VsnRaw) ->
Name = iolist_to_binary(NameRaw),
Vsn = iolist_to_binary(VsnRaw),
1 =< length([1 || {_, [AppName, AppVsn]} <- Warns,
AppName =:= Name, AppVsn =:= Vsn]).

+ 4
- 5
test/rebar_upgrade_SUITE.erl View File

@ -359,7 +359,7 @@ upgrades(delete_d) ->
top_level_deps([]) -> [];
top_level_deps([{{Name, Vsn, Ref}, _} | Deps]) ->
[{list_to_atom(Name), Vsn, Ref} | top_level_deps(Deps)];
top_level_deps([{{pkg, Name, Vsn, _URL}, _} | Deps]) ->
top_level_deps([{{pkg, Name, Vsn}, _} | Deps]) ->
[{list_to_atom(Name), Vsn} | top_level_deps(Deps)].
mock_deps(git, Deps, Upgrades) ->
@ -381,7 +381,7 @@ vsn_from_ref({git, _, {_, Vsn}}) -> Vsn;
vsn_from_ref({git, _, Vsn}) -> Vsn.
flat_pkgdeps([]) -> [];
flat_pkgdeps([{{pkg, Name, Vsn, _Url}, Deps} | Rest]) ->
flat_pkgdeps([{{pkg, Name, Vsn}, Deps} | Rest]) ->
[{{iolist_to_binary(Name),iolist_to_binary(Vsn)}, top_level_deps(Deps)}]
++
flat_pkgdeps(Deps)
@ -396,10 +396,10 @@ expand_deps(git, [{Name, Vsn, Deps} | Rest]) ->
Dep = {Name, Vsn, {git, "https://example.org/user/"++Name++".git", {tag, Vsn}}},
[{Dep, expand_deps(git, Deps)} | expand_deps(git, Rest)];
expand_deps(pkg, [{Name, Deps} | Rest]) ->
Dep = {pkg, Name, "0.0.0", "https://example.org/user/"++Name++".tar.gz"},
Dep = {pkg, Name, "0.0.0"},
[{Dep, expand_deps(pkg, Deps)} | expand_deps(pkg, Rest)];
expand_deps(pkg, [{Name, Vsn, Deps} | Rest]) ->
Dep = {pkg, Name, Vsn, "https://example.org/user/"++Name++".tar.gz"},
Dep = {pkg, Name, Vsn},
[{Dep, expand_deps(pkg, Deps)} | expand_deps(pkg, Rest)].
normalize_unlocks({App, Locks}) ->
@ -472,4 +472,3 @@ run(Config) ->
rebar_test_utils:run_and_check(
Config, NewRebarConfig, ["upgrade", App], Expectation
).

Loading…
Cancel
Save