From 31a24ad4ffb6a121819087da2cb9b58db5bd287d Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Wed, 19 Aug 2015 22:41:33 -0500 Subject: [PATCH 1/5] this patch treats pkg and src deps as equals, so level decides winner Instead fetching and resolving src deps (which could depend on pkg deps) and then pkg deps this patch combines the two into a single set of iterations by level. The only difference between src and pkg deps in this new install_deps is how their deps list is found -- from the config or lock file for src deps and from the neighbors of the vertex for pkg. --- src/rebar_app_discover.erl | 18 +- src/rebar_app_info.erl | 13 +- src/rebar_app_utils.erl | 129 +++++++++- src/rebar_digraph.erl | 110 --------- src/rebar_packages.erl | 26 +- src/rebar_prv_install_deps.erl | 395 +++++++----------------------- src/rebar_prv_packages.erl | 2 +- src/rebar_prv_upgrade.erl | 3 +- src/rebar_state.erl | 17 +- test/mock_pkg_resource.erl | 4 +- test/rebar_deps_SUITE.erl | 2 +- test/rebar_install_deps_SUITE.erl | 28 +-- 12 files changed, 275 insertions(+), 472 deletions(-) diff --git a/src/rebar_app_discover.erl b/src/rebar_app_discover.erl index e81a3237..8b1e58e3 100644 --- a/src/rebar_app_discover.erl +++ b/src/rebar_app_discover.erl @@ -96,7 +96,7 @@ merge_deps(AppInfo, State) -> {AppInfo1, State1}. handle_profile(Profile, Name, AppState, State) -> - {TopSrc, TopPkg} = rebar_state:get(State, {parsed_deps, Profile}, {[], []}), + TopParsedDeps = rebar_state:get(State, {parsed_deps, Profile}, {[], []}), TopLevelProfileDeps = rebar_state:get(State, {deps, Profile}, []), AppProfileDeps = rebar_state:get(AppState, {deps, Profile}, []), AppProfileDeps2 = rebar_utils:tup_dedup(rebar_utils:tup_sort(AppProfileDeps)), @@ -108,18 +108,18 @@ handle_profile(Profile, Name, AppState, State) -> %% Only deps not also specified in the top level config need %% to be included in the parsed deps NewDeps = ProfileDeps2 -- TopLevelProfileDeps, - {ParsedSrc, ParsedPkg} = parse_profile_deps(Profile, Name, NewDeps, AppState, State1), - rebar_state:set(State1, {parsed_deps, Profile}, {TopSrc++ParsedSrc, TopPkg++ParsedPkg}). + ParsedDeps = parse_profile_deps(Profile, Name, NewDeps, AppState, State1), + rebar_state:set(State1, {parsed_deps, Profile}, TopParsedDeps++ParsedDeps). parse_profile_deps(Profile, Name, Deps, AppState, State) -> DepsDir = rebar_prv_install_deps:profile_dep_dir(State, Profile), Locks = rebar_state:get(State, {locks, Profile}, []), - rebar_prv_install_deps:parse_deps(Name - ,DepsDir - ,Deps - ,AppState - ,Locks - ,1). + rebar_app_utils:parse_deps(Name + ,DepsDir + ,Deps + ,AppState + ,Locks + ,1). project_app_config(AppInfo, State) -> C = rebar_config:consult(rebar_app_info:dir(AppInfo)), diff --git a/src/rebar_app_info.erl b/src/rebar_app_info.erl index 7e31f6dd..bb99584d 100644 --- a/src/rebar_app_info.erl +++ b/src/rebar_app_info.erl @@ -34,6 +34,8 @@ dir/2, out_dir/1, out_dir/2, + resource_type/1, + resource_type/2, source/1, source/2, state/1, @@ -64,6 +66,7 @@ dep_level=0 :: integer(), dir :: file:name(), out_dir :: file:name(), + resource_type :: pkg | src, source :: string() | tuple() | undefined, state :: rebar_state:t() | undefined, is_lock=false :: boolean(), @@ -274,6 +277,14 @@ out_dir(AppInfo=#app_info_t{}, OutDir) -> ebin_dir(#app_info_t{out_dir=OutDir}) -> ec_cnv:to_list(filename:join(OutDir, "ebin")). +-spec resource_type(t(), pkg | src) -> t(). +resource_type(AppInfo=#app_info_t{}, Type) -> + AppInfo#app_info_t{resource_type=Type}. + +-spec resource_type(t()) -> pkg | src. +resource_type(#app_info_t{resource_type=ResourceType}) -> + ResourceType. + -spec source(t(), string() | tuple()) -> t(). source(AppInfo=#app_info_t{}, Source) -> AppInfo#app_info_t{source=Source}. @@ -316,7 +327,7 @@ is_checkout(#app_info_t{is_checkout=IsCheckout}) -> -spec valid(t()) -> boolean(). valid(AppInfo=#app_info_t{valid=undefined, state=State}) -> - case rebar_app_utils:validate_application_info(AppInfo) + case rebar_app_utils:validate_application_info(AppInfo) =:= true andalso rebar_state:has_all_artifacts(State) =:= true of true -> true; diff --git a/src/rebar_app_utils.erl b/src/rebar_app_utils.erl index e9745c35..17b435c5 100644 --- a/src/rebar_app_utils.erl +++ b/src/rebar_app_utils.erl @@ -32,6 +32,9 @@ app_src_to_app/2, validate_application_info/1, validate_application_info/2, + parse_deps/5, + parse_deps/6, + dep_to_app/7, format_error/1]). -include("rebar.hrl"). @@ -87,6 +90,109 @@ validate_application_info(AppInfo, AppDetail) -> end end. +-spec parse_deps(binary(), list(), rebar_state:t(), list(), integer()) -> {[rebar_app_info:t()], [tuple()]}. +parse_deps(DepsDir, Deps, State, Locks, Level) -> + parse_deps(root, DepsDir, Deps, State, Locks, Level). + +parse_deps(Parent, DepsDir, Deps, State, Locks, Level) -> + [parse_dep(Dep, Parent, DepsDir, State, Locks, Level) || Dep <- Deps]. + +parse_dep(Dep, Parent, DepsDir, State, Locks, Level) -> + Name = case Dep of + Dep when is_tuple(Dep) -> + element(1, Dep); + Dep -> + Dep + end, + case lists:keyfind(ec_cnv:to_binary(Name), 1, Locks) of + false -> + parse_dep(Parent, Dep, DepsDir, false, State); + LockedDep -> + LockedLevel = element(3, LockedDep), + case LockedLevel > Level of + true -> + parse_dep(Parent, Dep, DepsDir, false, State); + false -> + parse_dep(Parent, LockedDep, DepsDir, true, State) + end + end. + +parse_dep(Parent, {Name, Vsn}, DepsDir, IsLock, State) when is_list(Vsn); is_binary(Vsn) -> + %% Versioned Package dependency + CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)), + case rebar_app_info:discover(CheckoutsDir) of + {ok, _App} -> + dep_to_app(root, DepsDir, Name, [], [], IsLock, State); + not_found -> + {PkgName, PkgVsn} = parse_goal(ec_cnv:to_binary(Name) + ,ec_cnv:to_binary(Vsn)), + Source = {pkg, PkgName, PkgVsn}, + rebar_app_info:resource_type(dep_to_app(Parent, DepsDir, PkgName, PkgVsn, Source, IsLock, State), pkg) + end; +parse_dep(Parent, Name, DepsDir, IsLock, State) when is_atom(Name); is_binary(Name) -> + %% Unversioned package dependency + {PkgName, PkgVsn} = get_package(ec_cnv:to_binary(Name), State), + CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)), + case rebar_app_info:discover(CheckoutsDir) of + {ok, _App} -> + dep_to_app(root, DepsDir, Name, [], [], IsLock, State); + not_found -> + Source = {pkg, PkgName, PkgVsn}, + rebar_app_info:resource_type(dep_to_app(Parent, DepsDir, PkgName, PkgVsn, Source, IsLock, State), pkg) + end; +parse_dep(Parent, {Name, Source}, DepsDir, IsLock, State) when is_tuple(Source) -> + dep_to_app(Parent, DepsDir, Name, [], Source, IsLock, State); +parse_dep(Parent, {Name, _Vsn, Source}, DepsDir, IsLock, State) when is_tuple(Source) -> + dep_to_app(Parent, DepsDir, Name, [], Source, IsLock, State); +parse_dep(Parent, {Name, _Vsn, Source, Opts}, DepsDir, IsLock, State) when is_tuple(Source) -> + ?WARN("Dependency option list ~p in ~p is not supported and will be ignored", [Opts, Name]), + dep_to_app(Parent, DepsDir, Name, [], Source, IsLock, State); +parse_dep(Parent, {_Name, {pkg, Name, Vsn}, Level}, DepsDir, IsLock, State) when is_integer(Level) -> + CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)), + case rebar_app_info:discover(CheckoutsDir) of + {ok, _App} -> + dep_to_app(root, DepsDir, Name, [], [], IsLock, State); + not_found -> + Source = {pkg, Name, Vsn}, + rebar_app_info:resource_type(dep_to_app(Parent, DepsDir, Name, Vsn, Source, IsLock, State), pkg) + end; +parse_dep(Parent, {Name, Source, Level}, DepsDir, IsLock, State) when is_tuple(Source) + , is_integer(Level) -> + dep_to_app(Parent, DepsDir, Name, [], Source, IsLock, State); +parse_dep(_, Dep, _, _, _) -> + throw(?PRV_ERROR({parse_dep, Dep})). + +dep_to_app(Parent, DepsDir, Name, Vsn, Source, IsLock, State) -> + CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)), + BaseDir = rebar_state:get(State, base_dir, []), + {ok, Dep} = case rebar_app_info:discover(CheckoutsDir) of + {ok, App} -> + {ok, rebar_app_info:is_checkout(App, true)}; + not_found -> + Dir = ec_cnv:to_list(filename:join(DepsDir, Name)), + case rebar_app_info:discover(Dir) of + {ok, App} -> + {ok, App}; + not_found -> + rebar_app_info:new(Name, Vsn, + ec_cnv:to_list(filename:join(DepsDir, Name))) + end + end, + C = rebar_config:consult(rebar_app_info:dir(Dep)), + S = rebar_state:new(rebar_state:new(), C, rebar_app_info:dir(Dep)), + Overrides = rebar_state:get(State, overrides, []), + ParentOverrides = rebar_state:overrides(State), + S1 = rebar_state:set(rebar_state:overrides(S, ParentOverrides++Overrides), base_dir, BaseDir), + Dep1 = rebar_app_info:state(Dep, S1), + AppInfo = rebar_app_info:is_lock(rebar_app_info:source(Dep1, Source), IsLock), + ResourceType = case Source of + {pkg, _, _} -> + pkg; + _ -> + src + end, + rebar_app_info:resource_type(rebar_app_info:parent(AppInfo, Parent), ResourceType). + format_error(Error) -> io_lib:format("~p", [Error]). @@ -94,11 +200,30 @@ format_error(Error) -> %% Internal functions %% =================================================================== +-spec parse_goal(binary(), binary()) -> {binary(), binary()} | {binary(), binary(), binary()}. +parse_goal(Name, Constraint) -> + case re:run(Constraint, "([^\\d]*)(\\d.*)", [{capture, [1,2], binary}]) of + {match, [<<>>, Vsn]} -> + {Name, Vsn}; + {match, [Op, Vsn]} -> + {Name, Vsn, binary_to_atom(Op, utf8)}; + nomatch -> + throw(?PRV_ERROR({bad_constraint, Name, Constraint})) + end. + +get_package(Dep, State) -> + case rebar_state:registry(State) of + {ok, T} -> + {ok, HighestDepVsn} = rebar_packages:find_highest_matching(Dep, "0", T), + {Dep, HighestDepVsn}; + error -> + throw(?PRV_ERROR({load_registry_fail, Dep})) + end. + -spec has_all_beams(file:filename_all(), [module()]) -> true | ?PRV_ERROR({missing_module, module()}). has_all_beams(EbinDir, [Module | ModuleList]) -> - BeamFile = filename:join([EbinDir, - ec_cnv:to_list(Module) ++ ".beam"]), + BeamFile = filename:join([EbinDir, ec_cnv:to_list(Module) ++ ".beam"]), case filelib:is_file(BeamFile) of true -> has_all_beams(EbinDir, ModuleList); diff --git a/src/rebar_digraph.erl b/src/rebar_digraph.erl index ff0a1d28..363253a2 100644 --- a/src/rebar_digraph.erl +++ b/src/rebar_digraph.erl @@ -2,8 +2,6 @@ -export([compile_order/1 ,restore_graph/1 - ,cull_deps/2 - ,cull_deps/3 ,subgraph/2 ,format_error/1]). @@ -69,17 +67,6 @@ restore_graph({Vs, Es}) -> end, Es), Graph. -%% Pick packages to fullfill dependencies -%% The first dep while traversing the graph is chosen and any conflicting -%% dep encountered later on is ignored. -cull_deps(Graph, Vertices) -> - cull_deps(Graph, Vertices, sets:new()). - -cull_deps(Graph, Vertices, Seen) -> - Vertices1 = lists:keysort(2, Vertices), - {Solution, Levels, Discarded} = {dict:new(), dict:new(), sets:new()}, - cull_deps(Graph, Vertices1, Levels, Solution, Seen, Discarded). - format_error(no_solution) -> io_lib:format("No solution for packages found.", []). @@ -87,103 +74,6 @@ format_error(no_solution) -> %% Internal Functions %%==================================================================== -cull_deps(_Graph, [], Levels, Solution, _, Discarded) -> - {_, Vertices} = lists:unzip(dict:to_list(Solution)), - LvlVertices = [{Profile, {Parent, App, Vsn, dict:fetch(App, Levels)}} - || {Profile, {Parent,App,Vsn}} <- Vertices], - {ok, LvlVertices, sets:to_list(Discarded)}; -cull_deps(Graph, [{Profile, Level, Vs} | Vertices], Levels, Solution, Seen, Discarded) -> - {NV, NS, LS, DS} = - lists:foldl(fun({Parent, Name, Vsn}, {Acc, SolutionAcc, LevelsAcc, DiscardedAcc}) -> - {SolutionAcc1, LevelsAcc1, DiscardedAcc1} = - maybe_add_to_solution(Profile, Level, Name, {Name, Vsn}, Parent - ,SolutionAcc - ,LevelsAcc, Seen, DiscardedAcc), - OutNeighbors = digraph:out_neighbours(Graph, {Name,Vsn}), - {NewVertices, DiscardedAcc2} = handle_neighbors(Profile, Level, Name - ,OutNeighbors, Acc, SolutionAcc1 - ,Seen, DiscardedAcc1), - {NewVertices, SolutionAcc1, LevelsAcc1, DiscardedAcc2} - end, {[], Solution, Levels, Discarded}, Vs), - NewVertices = combine_profile_levels(Vertices, NV), - cull_deps(Graph, NewVertices, LS, NS, Seen, DS). - -%% Combine lists of deps that have the same profile and level -combine_profile_levels(Vertices, NewVertices) -> - V = lists:foldl(fun({Profile, Level, Vs}, Acc) -> - case ec_lists:find(fun({P, L, _}) -> - P =:= Profile andalso L =:= Level - end, Acc) of - {ok, {_, _, OldVs}=Old} -> - lists:delete(Old, Acc)++[{Profile, Level, lists:keysort(1, OldVs++Vs)}]; - error -> - Acc++[{Profile, Level, Vs}] - end - end, Vertices, NewVertices), - lists:keysort(2, V). - -%% For each outgoing edge of a dep check if it should be added to the solution -%% and add it to the list of vertices to do the same for -handle_neighbors(Profile, Level, Parent, OutNeighbors, Vertices - ,Solution, Seen, Discarded) -> - case lists:foldl(fun({Name, Vsn}=Value, {NewVertices, Discarded1}) -> - case dict:find(Name, Solution) of - {ok, {Profile, {Parent, Name, Vsn}}} -> % already seen - {NewVertices, - Discarded1}; - {ok, _} -> % conflict resolution! - %% Warn on different version - {NewVertices, - sets:add_element(Value, Discarded1)}; - error -> - %% We check Seen separately because we don't care - %% to warn if the exact same version of a package - %% was already part of the solution but we do - %% if it was simply seen in source deps - case sets:is_element(Name, Seen) of - true -> - {NewVertices, - sets:add_element(Value, Discarded1)}; - false -> - {[{Parent, Name, Vsn} | NewVertices], - Discarded1} - end - end - end, {[], Discarded}, OutNeighbors) of - {[], DiscardedAcc2} -> - {Vertices, DiscardedAcc2}; - {NewVertices1, DiscardedAcc2} -> - {Vertices++[{Profile, Level+1, NewVertices1}] ,DiscardedAcc2} - end. - -maybe_add_to_solution(Profile, Level, Key, {Name, Vsn}=Value, Parent - ,Solution, Levels, Seen, Discarded) -> - case dict:find(Key, Solution) of - {ok, {Profile, {Parent, Name, Vsn}}} -> % already seen - {Solution, - Levels, - Discarded}; - {ok, _} -> % conflict resolution! - %% Warn on different version - {Solution, - Levels, - sets:add_element(Value, Discarded)}; - error -> - %% We check Seen separately because we don't care to warn if the exact - %% same version of a package was already part of the solution but we do - %% if it was simply seen in source deps - case sets:is_element(Name, Seen) of - true -> - {Solution, - Levels, - sets:add_element(Value, Discarded)}; - false -> - {dict:store(Key, {Profile, {Parent, Name, Vsn}}, Solution), - dict:store(Key, Level, Levels), - Discarded} - end - end. - subgraph(Graph, Vertices) -> digraph_utils:subgraph(Graph, Vertices). diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl index ca3b6764..5f3d1c90 100644 --- a/src/rebar_packages.erl +++ b/src/rebar_packages.erl @@ -1,6 +1,7 @@ -module(rebar_packages). -export([packages/1 + ,packages_graph/1 ,registry/1 ,package_dir/1 ,check_registry/3 @@ -15,32 +16,43 @@ -type vsn() :: binary(). -type package() :: pkg_name() | {pkg_name(), vsn()}. --spec packages(rebar_state:t()) -> {rebar_dict(), rebar_digraph()}. +-spec packages(rebar_state:t()) -> rebar_dict(). %% DON'T USE IT! Use rebar_state:packages(State) instead. packages(State) -> RegistryDir = package_dir(State), DictFile = filename:join(RegistryDir, "dict"), + + try + {ok, DictBinary} = file:read_file(DictFile), + binary_to_term(DictBinary) + catch + _:_ -> + ?ERROR("Bad packages index, try to fix with `rebar3 update`", []), + dict:new() + end. + +-spec packages_graph(rebar_state:t()) -> rebar_digraph(). +packages_graph(State) -> + RegistryDir = package_dir(State), Edges = filename:join(RegistryDir, "edges"), Vertices = filename:join(RegistryDir, "vertices"), Neighbors = filename:join(RegistryDir, "neighbors"), - case lists:all(fun(X) -> filelib:is_file(X) end, [DictFile, Edges, Vertices, Neighbors]) of + case lists:all(fun(X) -> filelib:is_file(X) end, [Edges, Vertices, Neighbors]) of true -> try - {ok, DictBinary} = file:read_file(DictFile), - Dict = binary_to_term(DictBinary), {ok, EdgesTab} = ets:file2tab(Edges), {ok, VerticesTab} = ets:file2tab(Vertices), {ok, NeighborsTab} = ets:file2tab(Neighbors), - {Dict, {digraph, EdgesTab, VerticesTab, NeighborsTab, true}} + {digraph, EdgesTab, VerticesTab, NeighborsTab, true} catch _:_ -> ?ERROR("Bad packages index, try to fix with `rebar3 update`", []), - {dict:new(), digraph:new()} + digraph:new() end; false -> ?ERROR("Bad packages index, try to fix with `rebar3 update`", []), - {dict:new(), digraph:new()} + digraph:new() end. -spec registry(rebar_state:t()) -> {ok, ets:tid()} | {error, any()}. diff --git a/src/rebar_prv_install_deps.erl b/src/rebar_prv_install_deps.erl index 3a5a7cd7..0883fabe 100644 --- a/src/rebar_prv_install_deps.erl +++ b/src/rebar_prv_install_deps.erl @@ -36,8 +36,6 @@ -include_lib("providers/include/providers.hrl"). -export([handle_deps_as_profile/4, - parse_deps/5, - parse_deps/6, profile_dep_dir/2, find_cycles/1, cull_compile/2]). @@ -125,17 +123,12 @@ handle_deps_as_profile(Profile, State, Deps, Upgrade) -> Locks = [], Level = 0, DepsDir = profile_dep_dir(State, Profile), - {SrcDeps, PkgDeps} = parse_deps(DepsDir, Deps, State, Locks, Level), - AllSrcProfileDeps = [{Profile, SrcDeps, Locks, Level}], - AllPkgProfileDeps = case PkgDeps of - [] -> - []; - _ -> - [{Profile, Level, PkgDeps}] - end, - {AllApps, PkgDeps1, Seen, State1} = handle_profile_level(AllSrcProfileDeps, AllPkgProfileDeps, Locks, sets:new(), Upgrade, State), - - handle_profile_pkg_level(PkgDeps1, AllApps, Seen, Upgrade, [], State1). + Deps1 = rebar_app_utils:parse_deps(DepsDir, Deps, State, Locks, Level), + ProfileLevelDeps = [{Profile, Deps1, Level}], + Graph = rebar_state:packages_graph(State), + Registry = rebar_packages:registry(State), + State1 = rebar_state:packages_graph(rebar_state:registry(State, Registry), Graph), + handle_profile_level(ProfileLevelDeps, [], sets:new(), Upgrade, Locks, State1, Graph). %% =================================================================== %% Internal functions @@ -144,71 +137,33 @@ handle_deps_as_profile(Profile, State, Deps, Upgrade) -> %% finds all the deps in `{deps, ...}` for each profile provided. deps_per_profile(Profiles, Upgrade, State) -> Level = 0, - {AllProfileDeps, PkgDeps} = lists:foldl(fun(Profile, {SrcAcc, PkgAcc}) -> - case parse_profile_deps(State, Profile, Level) of - {Src, {_, _, []}} -> - {[Src | SrcAcc], PkgAcc}; - {Src, Pkg} -> - {[Src | SrcAcc], [Pkg | PkgAcc]} - end - end, {[], []}, Profiles), - {AllApps, PkgDeps1, Seen, State1} = handle_profile_level(AllProfileDeps, PkgDeps, [], sets:new(), Upgrade, State), Locks = rebar_state:get(State, {locks, default}, []), - handle_profile_pkg_level(PkgDeps1, AllApps, Seen, Upgrade, Locks, State1). - -parse_profile_deps(State, Profile, Level) -> - Locks = rebar_state:get(State, {locks, Profile}, []), - {SrcDeps, PkgDeps} = rebar_state:get(State, {parsed_deps, Profile}, {[], []}), - {{Profile, SrcDeps, Locks, Level}, {Profile, Level, PkgDeps}}. + Deps = lists:foldl(fun(Profile, DepAcc) -> + [parsed_profile_deps(State, Profile, Level) | DepAcc] + end, [], Profiles), + Graph = rebar_state:packages_graph(State), + Registry = rebar_packages:registry(State), + State1 = rebar_state:packages_graph(rebar_state:registry(State, Registry), Graph), + handle_profile_level(Deps, [], sets:new(), Upgrade, Locks, State1, Graph). + +parsed_profile_deps(State, Profile, Level) -> + ParsedDeps = rebar_state:get(State, {parsed_deps, Profile}, []), + {Profile, ParsedDeps, Level}. %% Level-order traversal of all dependencies, across profiles. %% If profiles x,y,z are present, then the traversal will go: %% x0, y0, z0, x1, y1, z1, ..., xN, yN, zN. -handle_profile_level([], PkgDeps, SrcApps, Seen, _Upgrade, State) -> - {SrcApps, PkgDeps, Seen, State}; -handle_profile_level([{Profile, SrcDeps, Locks, Level} | Rest], PkgDeps, SrcApps, Seen, Upgrade, State) -> - {SrcDeps1, PkgDeps1, SrcApps1, State1, Seen1, Locks1} = - update_src_deps(Profile, Level, SrcDeps, [], SrcApps - ,State, Upgrade, Seen, Locks), - SrcDeps2 = case SrcDeps1 of +handle_profile_level([], Apps, _Seen, _Upgrade, _Locks, State, _Graph) -> + {Apps, State}; +handle_profile_level([{Profile, Deps, Level} | Rest], Apps, Seen, Upgrade, Locks, State, Graph) -> + {Deps1, Apps1, State1, Seen1} = + update_deps(Profile, Level, Deps, Apps + ,State, Upgrade, Seen, Locks, Graph), + Deps2 = case Deps1 of [] -> Rest; - _ -> Rest ++ [{Profile, SrcDeps1, Locks1, Level+1}] + _ -> Rest ++ [{Profile, Deps1, Level+1}] end, - handle_profile_level(SrcDeps2, case PkgDeps1 of - [] -> - PkgDeps; - _ -> - [{Profile, Level+1, PkgDeps1} | PkgDeps] - end, SrcApps1, sets:union(Seen, Seen1), Upgrade, State1). - -handle_profile_pkg_level([], AllApps, _Seen, _Upgrade, _Locks, State) -> - {AllApps, State}; -handle_profile_pkg_level(PkgDeps, AllApps, Seen, Upgrade, Locks, State) -> - %% Read in package index and dep graph - {Packages, Graph} = rebar_state:packages(State), - Registry = rebar_state:registry(State), - State1 = rebar_state:packages(rebar_state:registry(State, Registry) - ,{Packages, Graph}), - - S = case rebar_digraph:cull_deps(Graph, lists:keysort(2, PkgDeps), Seen) of - {ok, [], []} -> - throw({rebar_digraph, no_solution}); - {ok, [], Discarded} -> - [warn_skip_pkg(Pkg, State) || Upgrade =:= false, - Pkg <- Discarded, - not(pkg_locked(Pkg, Locks))], - []; - {ok, Solution, []} -> - Solution; - {ok, Solution, Discarded} -> - [warn_skip_pkg(Pkg, State) || Upgrade =:= false, - Pkg <- Discarded, - not(pkg_locked(Pkg, Locks))], - Solution - end, - - {PkgApps, State2} = update_pkg_deps(S, Packages, Upgrade, Seen, State1, Locks), - {AllApps++PkgApps, State2}. + handle_profile_level(Deps2, Apps1, sets:union(Seen, Seen1), Upgrade, Locks, State1, Graph). find_cycles(Apps) -> case rebar_digraph:compile_order(Apps) of @@ -220,38 +175,6 @@ find_cycles(Apps) -> cull_compile(TopSortedDeps, ProjectApps) -> lists:dropwhile(fun not_needs_compile/1, TopSortedDeps -- ProjectApps). -pkg_locked({_, Name, _, _}, Locks) -> - pkg_locked(Name, Locks); -pkg_locked({Name, _}, Locks) -> - pkg_locked(Name, Locks); -pkg_locked(Name, Locks) -> - false =/= lists:keyfind(Name, 1, Locks). - -update_pkg_deps(Pkgs, Packages, Upgrade, Seen, State, Locks) -> - {Solved, _, State1} - = lists:foldl(fun({Profile, Pkg}, {Acc, SeenAcc, StateAcc}) -> - DepsDir = profile_dep_dir(State, Profile), - handle_pkg_dep(Profile, Pkg, Packages, Upgrade, DepsDir, Acc, SeenAcc, Locks, StateAcc) - end, {[], Seen, State}, Pkgs), - {Solved, State1}. - -handle_pkg_dep(Profile, Pkg, Packages, Upgrade, DepsDir, Fetched, Seen, Locks, State) -> - IsLock = pkg_locked(Pkg, Locks), - AppInfo = package_to_app(DepsDir, Packages, Pkg, IsLock, State), - case sets:is_element(rebar_app_info:name(AppInfo), Seen) of - true -> - {Fetched, Seen, State}; - false -> - Deps = rebar_app_info:deps(AppInfo), - Level = rebar_app_info:dep_level(AppInfo), - {NewSeen, NewState} = maybe_lock(Profile, AppInfo, Seen, State, Level), - {_, AppInfo1} = maybe_fetch(AppInfo, Profile, Upgrade, Seen, NewState), - {AppInfo2, _, _, _, _} = - handle_dep(NewState, Profile, DepsDir, AppInfo1, Locks, Level), - AppInfo3 = rebar_app_info:deps(AppInfo2, Deps), - {[AppInfo3 | Fetched], NewSeen, NewState} - end. - maybe_lock(Profile, AppInfo, Seen, State, Level) -> Name = rebar_app_info:name(AppInfo), case rebar_app_info:is_checkout(AppInfo) of @@ -278,49 +201,28 @@ maybe_lock(Profile, AppInfo, Seen, State, Level) -> {sets:add_element(Name, Seen), State} end. -package_to_app(DepsDir, Packages, {Parent, Name, Vsn, Level}, IsLock, State) -> - case dict:find({Name, Vsn}, Packages) of - error -> - case rebar_packages:check_registry(Name, Vsn, State) of - true -> - throw(?PRV_ERROR({not_rebar_package, Name, Vsn})); - false -> - throw(?PRV_ERROR({missing_package, Name, Vsn})) - end; - {ok, PkgDeps} -> - Source = {pkg, Name, Vsn}, - AppInfo = new_dep(root, DepsDir, Name, Vsn, Source, IsLock, State), - AppInfo1 = rebar_app_info:dep_level(rebar_app_info:deps(AppInfo, PkgDeps), Level), - BaseDir = rebar_state:get(State, base_dir, []), - AppState1 = rebar_state:set(rebar_app_info:state(AppInfo1), base_dir, BaseDir), - rebar_app_info:parent(rebar_app_info:state(AppInfo1, AppState1), Parent) - end. - --spec update_src_deps(atom(), non_neg_integer(), list(), list(), list(), rebar_state:t(), boolean(), sets:set(binary()), list()) -> {list(), list(), list(), rebar_state:t(), sets:set(binary()), list()}. -update_src_deps(Profile, Level, SrcDeps, PkgDeps, SrcApps, State, Upgrade, Seen, Locks) -> +update_deps(Profile, Level, Deps, Apps, State, Upgrade, Seen, Locks, Graph) -> lists:foldl( - fun(AppInfo, {SrcDepsAcc, PkgDepsAcc, SrcAppsAcc, StateAcc, SeenAcc, LocksAcc}) -> - update_src_dep(AppInfo, Profile, Level, - SrcDepsAcc, PkgDepsAcc, SrcAppsAcc, StateAcc, - Upgrade, SeenAcc, Locks, LocksAcc) + fun(AppInfo, {DepsAcc, AppsAcc, StateAcc, SeenAcc}) -> + update_dep(AppInfo, Profile, Level, + DepsAcc, AppsAcc, StateAcc, + Upgrade, SeenAcc, Locks, Graph) end, - {[], PkgDeps, SrcApps, State, Seen, Locks}, - rebar_utils:sort_deps(SrcDeps)). + {[], Apps, State, Seen}, + rebar_utils:sort_deps(Deps)). - -update_src_dep(AppInfo, Profile, Level, SrcDeps, PkgDeps, SrcApps, State, Upgrade, Seen, BaseLocks, Locks) -> +update_dep(AppInfo, Profile, Level, Deps, Apps, State, Upgrade, Seen, Locks, Graph) -> %% If not seen, add to list of locks to write out Name = rebar_app_info:name(AppInfo), case sets:is_element(Name, Seen) of true -> - update_seen_src_dep(AppInfo, Profile, Level, - SrcDeps, PkgDeps, SrcApps, - State, Upgrade, Seen, BaseLocks, Locks); + update_seen_dep(AppInfo, Profile, Level, + Deps, Apps, + State, Upgrade, Seen, Locks); false -> - update_unseen_src_dep(AppInfo, Profile, Level, - SrcDeps, PkgDeps, SrcApps, - State, Upgrade, Seen, Locks) - + update_unseen_dep(AppInfo, Profile, Level, + Deps, Apps, + State, Upgrade, Seen, Locks, Graph) end. profile_dep_dir(State, Profile) -> @@ -329,41 +231,31 @@ profile_dep_dir(State, Profile) -> _ -> rebar_dir:deps_dir(State) end. -update_seen_src_dep(AppInfo, _Profile, _Level, SrcDeps, PkgDeps, SrcApps, State, Upgrade, Seen, BaseLocks, Locks) -> +update_seen_dep(AppInfo, _Profile, _Level, Deps, Apps, State, Upgrade, Seen, Locks) -> Name = rebar_app_info:name(AppInfo), %% If seen from lock file or user requested an upgrade %% don't print warning about skipping - case lists:keymember(Name, 1, BaseLocks) of + case lists:keymember(Name, 1, Locks) of false when Upgrade -> ok; false when not Upgrade -> warn_skip_deps(AppInfo, State); true -> ok end, - {SrcDeps, PkgDeps, SrcApps, State, Seen, Locks}. + {Deps, Apps, State, Seen}. -update_unseen_src_dep(AppInfo, Profile, Level, SrcDeps, PkgDeps, SrcApps, State, Upgrade, Seen, Locks) -> +update_unseen_dep(AppInfo, Profile, Level, Deps, Apps, State, Upgrade, Seen, Locks, Graph) -> {NewSeen, State1} = maybe_lock(Profile, AppInfo, Seen, State, Level), {_, AppInfo1} = maybe_fetch(AppInfo, Profile, Upgrade, Seen, State1), - {NewSrcDeps, NewPkgDeps, NewSrcApps, State2, NewLocks} = - handle_dep(AppInfo1, Profile, SrcDeps, PkgDeps, SrcApps, - Level, State1, Locks), - {NewSrcDeps, NewPkgDeps, NewSrcApps, State2, NewSeen, NewLocks}. - -handle_dep(AppInfo, Profile, SrcDeps, PkgDeps, SrcApps, Level, State, Locks) -> DepsDir = profile_dep_dir(State, Profile), - {AppInfo1, NewSrcDeps, NewPkgDeps, NewLocks, State1} = - handle_dep(State, Profile, DepsDir, AppInfo, Locks, Level), - AppInfo2 = rebar_app_info:dep_level(AppInfo1, Level), - {NewSrcDeps ++ SrcDeps - ,NewPkgDeps++PkgDeps - ,[AppInfo2 | SrcApps] - ,State1 - ,NewLocks}. - --spec handle_dep(rebar_state:t(), atom(), file:filename_all(), rebar_app_info:t(), list(), integer()) -> - {rebar_app_info:t(), [rebar_app_info:t()], [pkg_dep()], [integer()], rebar_state:t()}. -handle_dep(State, Profile, DepsDir, AppInfo, Locks, Level) -> + {AppInfo2, NewDeps, State2} = + handle_dep(State1, Profile, DepsDir, AppInfo1, Locks, Level, Graph), + AppInfo3 = rebar_app_info:dep_level(AppInfo2, Level), + {NewDeps ++ Deps, [AppInfo3 | Apps], State2, NewSeen}. + +-spec handle_dep(rebar_state:t(), atom(), file:filename_all(), rebar_app_info:t(), list(), integer(), rebar_dict()) -> {rebar_app_info:t(), [rebar_app_info:t()], [pkg_dep()], [integer()], rebar_state:t()}. +handle_dep(State, Profile, DepsDir, AppInfo, Locks, Level, Graph) -> Profiles = rebar_state:current_profiles(State), Name = rebar_app_info:name(AppInfo), + Vsn = rebar_app_info:original_vsn(AppInfo), %% Deps may be under a sub project app, find it and use its state if so S0 = rebar_app_info:state(AppInfo), @@ -375,23 +267,26 @@ handle_dep(State, Profile, DepsDir, AppInfo, Locks, Level) -> S3 = rebar_state:apply_profiles(S2, Profiles), Plugins = rebar_state:get(S3, plugins, []), S4 = rebar_state:set(S3, {plugins, Profile}, Plugins), - AppInfo1 = rebar_app_info:state(AppInfo, S4), rebar_utils:check_min_otp_version(rebar_state:get(S4, minimum_otp_vsn, undefined)), rebar_utils:check_blacklisted_otp_versions(rebar_state:get(S4, blacklisted_otp_vsns, [])), %% Dep may have plugins to install. Find and install here. S5 = rebar_plugins:install(S4), - AppInfo2 = rebar_app_info:state(AppInfo1, S5), + AppInfo1 = rebar_app_info:state(AppInfo, S5), %% Upgrade lock level to be the level the dep will have in this dep tree - Deps = rebar_state:get(S5, {deps, default}, []), - NewLocks = Locks++[{DepName, Source, LockLevel+Level} || - {DepName, Source, LockLevel} <- rebar_state:get(S5, {locks, default}, [])], - AppInfo3 = rebar_app_info:deps(AppInfo2, rebar_state:deps_names(Deps)), - {SrcDeps, PkgDeps} = parse_deps(rebar_app_info:name(AppInfo3), DepsDir, Deps - ,S5, NewLocks, Level+1), - {AppInfo3, SrcDeps, PkgDeps, NewLocks, State}. + case rebar_app_info:resource_type(AppInfo1) of + pkg -> + NewDeps = digraph:out_neighbours(Graph, {ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)}), + NewDeps1 = rebar_app_utils:parse_deps(Name, DepsDir, NewDeps, S5, Locks, Level+1), + {rebar_app_info:deps(AppInfo1, NewDeps), NewDeps1, State}; + _ -> + Deps = rebar_state:get(S5, {deps, default}, []), + AppInfo2 = rebar_app_info:deps(AppInfo1, rebar_state:deps_names(Deps)), + Deps1 = rebar_app_utils:parse_deps(Name, DepsDir, Deps, S5, Locks, Level+1), + {AppInfo2, Deps1, State} + end. -spec maybe_fetch(rebar_app_info:t(), atom(), boolean(), sets:set(binary()), rebar_state:t()) -> {boolean(), rebar_app_info:t()}. @@ -406,20 +301,20 @@ maybe_fetch(AppInfo, Profile, Upgrade, Seen, State) -> false -> true = fetch_app(AppInfo, AppDir, State), maybe_symlink_default(State, Profile, AppDir, AppInfo), - {true, update_app_info(AppDir, AppInfo)}; + {true, rebar_app_info:valid(update_app_info(AppDir, AppInfo), false)}; {true, AppInfo1} -> %% Preserve the state we created with overrides + AppInfo2 = copy_app_info(AppInfo, AppInfo1), AppState = rebar_app_info:state(AppInfo), - Parent = rebar_app_info:parent(AppInfo), - Source = rebar_app_info:source(AppInfo), - AppInfo2 = rebar_app_info:parent(rebar_app_info:state(AppInfo1, AppState), Parent), - AppInfo3 = rebar_app_info:source(AppInfo2, Source), - case sets:is_element(rebar_app_info:name(AppInfo), Seen) of + AppInfo3 = rebar_app_info:state(AppInfo2, AppState), + case sets:is_element(rebar_app_info:name(AppInfo3), Seen) of true -> {false, AppInfo3}; false -> maybe_symlink_default(State, Profile, AppDir, AppInfo3), - {maybe_upgrade(AppInfo, AppDir, Upgrade, State), AppInfo3} + MaybeUpgrade = maybe_upgrade(AppInfo, AppDir, Upgrade, State), + AppInfo4 = update_app_info(AppDir, AppInfo3), + {MaybeUpgrade, AppInfo4} end end end. @@ -467,106 +362,6 @@ make_relative_to_root(State, Path) when is_list(Path) -> Root = rebar_dir:root_dir(State), rebar_dir:make_relative_path(Path, Root). --spec parse_deps(binary(), list(), rebar_state:t(), list(), integer()) -> {[rebar_app_info:t()], [tuple()]}. -parse_deps(DepsDir, Deps, State, Locks, Level) -> - parse_deps(root, DepsDir, Deps, State, Locks, Level). - -parse_deps(Parent, DepsDir, Deps, State, Locks, Level) -> - lists:foldl(fun(Dep, Acc) -> - Name = case Dep of - Dep when is_tuple(Dep) -> - element(1, Dep); - Dep -> - Dep - end, - case lists:keyfind(ec_cnv:to_binary(Name), 1, Locks) of - false -> - parse_dep(Parent, Dep, Acc, DepsDir, false, State); - LockedDep -> - LockedLevel = element(3, LockedDep), - case LockedLevel > Level of - true -> - parse_dep(Parent, Dep, Acc, DepsDir, false, State); - false -> - parse_dep(Parent, LockedDep, Acc, DepsDir, true, State) - end - end - end, {[], []}, Deps). - -parse_dep(Parent, {Name, Vsn}, {SrcDepsAcc, PkgDepsAcc}, DepsDir, IsLock, State) when is_list(Vsn) -> - %% Versioned Package dependency - CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)), - case rebar_app_info:discover(CheckoutsDir) of - {ok, _App} -> - Dep = new_dep(root, DepsDir, Name, [], [], IsLock, State), - {[Dep | SrcDepsAcc], PkgDepsAcc}; - not_found -> - {SrcDepsAcc, [parse_goal(Parent - ,ec_cnv:to_binary(Name) - ,ec_cnv:to_binary(Vsn)) | PkgDepsAcc]} - end; -parse_dep(Parent, Name, {SrcDepsAcc, PkgDepsAcc}, DepsDir, IsLock, State) when is_atom(Name); is_binary(Name) -> - %% Unversioned package dependency - {PkgName, PkgVsn} = get_package(ec_cnv:to_binary(Name), State), - CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)), - case rebar_app_info:discover(CheckoutsDir) of - {ok, _App} -> - Dep = new_dep(root, DepsDir, Name, [], [], IsLock, State), - {[Dep | SrcDepsAcc], PkgDepsAcc}; - not_found -> - {SrcDepsAcc, [{Parent, PkgName, PkgVsn} | PkgDepsAcc]} - end; -parse_dep(Parent, {Name, Source}, {SrcDepsAcc, PkgDepsAcc}, DepsDir, IsLock, State) when is_tuple(Source) -> - Dep = new_dep(Parent, DepsDir, Name, [], Source, IsLock, State), - {[Dep | SrcDepsAcc], PkgDepsAcc}; -parse_dep(Parent, {Name, _Vsn, Source}, {SrcDepsAcc, PkgDepsAcc}, DepsDir, IsLock, State) when is_tuple(Source) -> - Dep = new_dep(Parent, DepsDir, Name, [], Source, IsLock, State), - {[Dep | SrcDepsAcc], PkgDepsAcc}; -parse_dep(Parent, {Name, _Vsn, Source, Opts}, {SrcDepsAcc, PkgDepsAcc}, DepsDir, IsLock, State) when is_tuple(Source) -> - ?WARN("Dependency option list ~p in ~p is not supported and will be ignored", [Opts, Name]), - Dep = new_dep(Parent, DepsDir, Name, [], Source, IsLock, State), - {[Dep | SrcDepsAcc], PkgDepsAcc}; -parse_dep(Parent, {_Name, {pkg, Name, Vsn}, Level}, {SrcDepsAcc, PkgDepsAcc}, DepsDir, IsLock, State) when is_integer(Level) -> - CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)), - case rebar_app_info:discover(CheckoutsDir) of - {ok, _App} -> - Dep = new_dep(root, DepsDir, Name, [], [], IsLock, State), - {[Dep | SrcDepsAcc], PkgDepsAcc}; - not_found -> - {SrcDepsAcc, [{Parent, Name, Vsn} | PkgDepsAcc]} - end; -parse_dep(Parent, {Name, Source, Level}, {SrcDepsAcc, PkgDepsAcc}, DepsDir, IsLock, State) when is_tuple(Source) - , is_integer(Level) -> - Dep = new_dep(Parent, DepsDir, Name, [], Source, IsLock, State), - {[Dep | SrcDepsAcc], PkgDepsAcc}; -parse_dep(_, Dep, _, _, _, _) -> - throw(?PRV_ERROR({parse_dep, Dep})). - - -new_dep(Parent, DepsDir, Name, Vsn, Source, IsLock, State) -> - CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)), - {ok, Dep} = case rebar_app_info:discover(CheckoutsDir) of - {ok, App} -> - {ok, rebar_app_info:is_checkout(App, true)}; - not_found -> - Dir = ec_cnv:to_list(filename:join(DepsDir, Name)), - case rebar_app_info:discover(Dir) of - {ok, App} -> - {ok, App}; - not_found -> - rebar_app_info:new(Name, Vsn, - ec_cnv:to_list(filename:join(DepsDir, Name))) - end - end, - C = rebar_config:consult(rebar_app_info:dir(Dep)), - S = rebar_state:new(rebar_state:new(), C, rebar_app_info:dir(Dep)), - Overrides = rebar_state:get(State, overrides, []), - ParentOverrides = rebar_state:overrides(State), - Dep1 = rebar_app_info:state(Dep, - rebar_state:overrides(S, ParentOverrides++Overrides)), - AppInfo = rebar_app_info:is_lock(rebar_app_info:source(Dep1, Source), IsLock), - rebar_app_info:parent(AppInfo, Parent). - fetch_app(AppInfo, AppDir, State) -> ?INFO("Fetching ~s (~p)", [rebar_app_info:name(AppInfo), rebar_app_info:source(AppInfo)]), Source = rebar_app_info:source(AppInfo), @@ -577,15 +372,24 @@ fetch_app(AppInfo, AppDir, State) -> %% be read in an parsed. update_app_info(AppDir, AppInfo) -> {ok, Found} = rebar_app_info:discover(AppDir), - Parent = rebar_app_info:parent(AppInfo), - Source = rebar_app_info:source(AppInfo), AppDetails = rebar_app_info:app_details(Found), + Vsn = rebar_app_info:original_vsn(Found), Applications = proplists:get_value(applications, AppDetails, []), IncludedApplications = proplists:get_value(included_applications, AppDetails, []), - AppInfo1 = rebar_app_info:applications( + AppInfo1 = rebar_app_info:original_vsn(rebar_app_info:applications( rebar_app_info:app_details(AppInfo, AppDetails), - IncludedApplications++Applications), - rebar_app_info:source(rebar_app_info:parent(rebar_app_info:valid(AppInfo1, false), Parent), Source). + IncludedApplications++Applications), Vsn), + AppInfo2 = copy_app_info(AppInfo, AppInfo1), + rebar_app_info:valid(AppInfo2, undefined). + +copy_app_info(OldAppInfo, NewAppInfo) -> + ResourceType = rebar_app_info:resource_type(OldAppInfo), + Parent = rebar_app_info:parent(OldAppInfo), + Source = rebar_app_info:source(OldAppInfo), + + rebar_app_info:resource_type( + rebar_app_info:source( + rebar_app_info:parent(NewAppInfo, Parent), Source), ResourceType). maybe_upgrade(AppInfo, AppDir, Upgrade, State) -> Source = rebar_app_info:source(AppInfo), @@ -608,17 +412,6 @@ maybe_upgrade(AppInfo, AppDir, Upgrade, State) -> false end. --spec parse_goal(binary() | root, binary(), binary()) -> {binary(), binary()} | {binary(), binary(), binary()}. -parse_goal(Parent, Name, Constraint) -> - case re:run(Constraint, "([^\\d]*)(\\d.*)", [{capture, [1,2], binary}]) of - {match, [<<>>, Vsn]} -> - {Parent, Name, Vsn}; - {match, [Op, Vsn]} -> - {Parent, Name, Vsn, binary_to_atom(Op, utf8)}; - nomatch -> - throw(?PRV_ERROR({bad_constraint, Name, Constraint})) - end. - warn_skip_deps(AppInfo, State) -> Msg = "Skipping ~s (from ~p) as an app of the same name " "has already been fetched", @@ -629,25 +422,7 @@ warn_skip_deps(AppInfo, State) -> true -> ?ERROR(Msg, Args), ?FAIL end. -warn_skip_pkg({Name, Source}, State) -> - Msg = "Skipping ~s (version ~s from package index) as an app of the same " - "name has already been fetched", - Args = [Name, Source], - case rebar_state:get(State, deps_error_on_conflict, false) of - false -> ?WARN(Msg, Args); - true -> ?ERROR(Msg, Args), ?FAIL - end. - not_needs_compile(App) -> not(rebar_app_info:is_checkout(App)) andalso rebar_app_info:valid(App) andalso rebar_state:has_all_artifacts(rebar_app_info:state(App)) =:= true. - -get_package(Dep, State) -> - case rebar_state:registry(State) of - {ok, T} -> - {ok, HighestDepVsn} = rebar_packages:find_highest_matching(Dep, "0", T), - {Dep, HighestDepVsn}; - error -> - throw(?PRV_ERROR({load_registry_fail, Dep})) - end. diff --git a/src/rebar_prv_packages.erl b/src/rebar_prv_packages.erl index 82ed2f7e..44cde4e1 100644 --- a/src/rebar_prv_packages.erl +++ b/src/rebar_prv_packages.erl @@ -27,7 +27,7 @@ init(State) -> -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. do(State) -> - {Dict, _} = rebar_state:packages(State), + Dict = rebar_packages:packages(State), print_packages(Dict), {ok, State}. diff --git a/src/rebar_prv_upgrade.erl b/src/rebar_prv_upgrade.erl index f49eafe3..7385cfe5 100644 --- a/src/rebar_prv_upgrade.erl +++ b/src/rebar_prv_upgrade.erl @@ -54,7 +54,7 @@ do(State) -> Deps0 = top_level_deps(Deps, Locks), State1 = rebar_state:set(State, {deps, default}, Deps0), DepsDir = rebar_prv_install_deps:profile_dep_dir(State, default), - D = rebar_prv_install_deps:parse_deps(root, DepsDir, Deps0, State1, Locks0, 0), + D = rebar_app_utils:parse_deps(root, DepsDir, Deps0, State1, Locks0, 0), State2 = rebar_state:set(State1, {parsed_deps, default}, D), State3 = rebar_state:set(State2, {locks, default}, Locks0), State4 = rebar_state:set(State3, upgrade, true), @@ -83,7 +83,6 @@ format_error({transitive_dependency, Name}) -> format_error(Reason) -> io_lib:format("~p", [Reason]). - parse_names(Bin, Locks) -> case lists:usort(re:split(Bin, <<" *, *">>, [trim])) of %% Nothing submitted, use *all* apps diff --git a/src/rebar_state.erl b/src/rebar_state.erl index 7616151a..54e7f131 100644 --- a/src/rebar_state.erl +++ b/src/rebar_state.erl @@ -38,7 +38,8 @@ overrides/1, overrides/2, apply_overrides/2, - packages/1, packages/2, + packages_graph/1, packages_graph/2, + packages/1, registry/1, registry/2, resources/1, resources/2, add_resource/2, @@ -65,7 +66,8 @@ all_plugin_deps = [] :: [rebar_app_info:t()], all_deps = [] :: [rebar_app_info:t()], - packages = undefined :: {rebar_dict(), rebar_digraph()} | undefined, + packages = undefined :: rebar_dict(), + packages_graph = undefined :: rebar_digraph() | undefined, registry = undefined :: {ok, ets:tid()} | error | undefined, overrides = [], resources = [], @@ -96,7 +98,9 @@ new(Config) when is_list(Config) -> load_package_registry(Config0) -> Registry = rebar_packages:registry(Config0), Packages = rebar_packages:packages(Config0), + PackagesGraph = rebar_packages:packages_graph(Config0), Config0#state_t{registry = Registry, + packages_graph = PackagesGraph, packages = Packages}. -spec new(t() | atom(), list()) -> t(). @@ -451,8 +455,13 @@ packages(#state_t{packages=undefined}) -> packages(#state_t{packages=Packages}) -> Packages. -packages(State, Packages) -> - State#state_t{packages=Packages}. +packages_graph(#state_t{packages_graph=undefined}) -> + throw(packages_usage_error); +packages_graph(#state_t{packages_graph=PackagesGraph}) -> + PackagesGraph. + +packages_graph(State, PackagesGraph) -> + State#state_t{packages_graph=PackagesGraph}. registry(#state_t{registry=undefined}) -> throw(registry_usage_error); diff --git a/test/mock_pkg_resource.erl b/test/mock_pkg_resource.erl index e5c5645d..9258f72d 100644 --- a/test/mock_pkg_resource.erl +++ b/test/mock_pkg_resource.erl @@ -119,7 +119,9 @@ mock_pkg_index(Opts) -> meck:expect(rebar_state, registry, fun(_State) -> {ok, to_registry(Deps)} end), meck:expect(rebar_state, packages, - fun(_State) -> {Dict, Digraph} end). + fun(_State) -> Dict end), + meck:expect(rebar_state, packages_graph, + fun(_State) -> Digraph end). %%%%%%%%%%%%%%% %%% Helpers %%% diff --git a/test/rebar_deps_SUITE.erl b/test/rebar_deps_SUITE.erl index 04018fcd..fd86226f 100644 --- a/test/rebar_deps_SUITE.erl +++ b/test/rebar_deps_SUITE.erl @@ -297,5 +297,5 @@ in_warnings(git, Warns, NameRaw, VsnRaw) -> in_warnings(pkg, Warns, NameRaw, VsnRaw) -> Name = iolist_to_binary(NameRaw), Vsn = iolist_to_binary(VsnRaw), - 1 =< length([1 || {_, [AppName, AppVsn]} <- Warns, + 1 =< length([1 || {_, [AppName, {pkg, _, AppVsn}]} <- Warns, AppName =:= Name, AppVsn =:= Vsn]). diff --git a/test/rebar_install_deps_SUITE.erl b/test/rebar_install_deps_SUITE.erl index 66c762e9..9d536d11 100644 --- a/test/rebar_install_deps_SUITE.erl +++ b/test/rebar_install_deps_SUITE.erl @@ -15,9 +15,9 @@ groups() -> {git, [], [{group, unique}]}, {pkg, [], [{group, unique}]}, {mixed, [], [ - m_flat1, m_flat2, m_circular1, m_circular2, m_circular3, + m_flat1, m_flat2, m_circular1, m_circular2, m_pick_source1, m_pick_source2, m_pick_source3, - m_pick_source4, m_pick_source5, m_source_to_pkg, + m_source_to_pkg, m_pkg_level1, m_pkg_level2, m_pkg_level3, m_pkg_level3_alpha_order ]} ]. @@ -199,39 +199,22 @@ mdeps(m_circular2) -> {[{"B", [{"c", [{"b", []}]}]}], [], {error, {rebar_prv_install_deps, {cycles, [[<<"B">>,<<"C">>]]}}}}; -mdeps(m_circular3) -> - %% Spot the circular dep due to being to low in the deps tree - %% but as a source dep, taking precedence over packages - {[{"B", [{"C", "2", [{"B", []}]}]}, - {"c", "1", [{"d",[]}]}], - [], - {error, {rebar_prv_install_deps, {cycles, [[<<"B">>,<<"C">>]]}}}}; mdeps(m_pick_source1) -> {[{"B", [{"D", []}]}, {"c", [{"d", []}]}], ["d"], {ok, ["B", "c", "D"]}}; mdeps(m_pick_source2) -> - {[{"b", [{"d", []}]}, - {"C", [{"D", []}]}], - ["d"], - {ok, ["b", "C", "D"]}}; -mdeps(m_pick_source3) -> %% The order of declaration is important. {[{"b", []}, {"B", []}], [], {ok, ["b"]}}; -mdeps(m_pick_source4) -> +mdeps(m_pick_source3) -> {[{"B", []}, {"b", []}], [], {ok, ["B"]}}; -mdeps(m_pick_source5) -> - {[{"B", [{"d", []}]}, - {"C", [{"D", []}]}], - ["d"], - {ok, ["B", "C", "D"]}}; mdeps(m_source_to_pkg) -> {[{"B", [{"c",[{"d", []}]}]}], [], @@ -438,12 +421,9 @@ m_flat1(Config) -> run(Config). m_flat2(Config) -> run(Config). m_circular1(Config) -> run(Config). m_circular2(Config) -> run(Config). -m_circular3(Config) -> run(Config). m_pick_source1(Config) -> run(Config). m_pick_source2(Config) -> run(Config). m_pick_source3(Config) -> run(Config). -m_pick_source4(Config) -> run(Config). -m_pick_source5(Config) -> run(Config). m_source_to_pkg(Config) -> run(Config). m_pkg_level1(Config) -> run(Config). m_pkg_level2(Config) -> run(Config). @@ -483,5 +463,5 @@ in_warnings(git, Warns, NameRaw, VsnRaw) -> in_warnings(pkg, Warns, NameRaw, VsnRaw) -> Name = iolist_to_binary(NameRaw), Vsn = iolist_to_binary(VsnRaw), - 1 =< length([1 || {_, [AppName, AppVsn]} <- Warns, + 1 =< length([1 || {_, [AppName, {pkg, _, AppVsn}]} <- Warns, AppName =:= Name, AppVsn =:= Vsn]). From 6242c8155499df2fe141772830565b041959c7b9 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Fri, 21 Aug 2015 08:56:49 -0500 Subject: [PATCH 2/5] include rebar3 in supported packages buildtool --- src/rebar_prv_update.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/rebar_prv_update.erl b/src/rebar_prv_update.erl index 55c70e2a..e73c7693 100644 --- a/src/rebar_prv_update.erl +++ b/src/rebar_prv_update.erl @@ -84,6 +84,7 @@ write_registry(Dict, {digraph, Edges, Vertices, Neighbors, _}, State) -> is_supported(<<"make">>) -> true; is_supported(<<"rebar">>) -> true; +is_supported(<<"rebar3">>) -> true; is_supported(_) -> false. hex_to_graph(Filename) -> From cf5390f01876ff8d9e70cff521740ab0dd805929 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Fri, 21 Aug 2015 12:18:30 -0500 Subject: [PATCH 3/5] replace use of dict of packages and registry with single ets table --- src/rebar.hrl | 6 ++ src/rebar3.erl | 13 ++- src/rebar_app_utils.erl | 9 +- src/rebar_packages.erl | 168 ++++++++++++++++----------------- src/rebar_plugins.erl | 10 +- src/rebar_prv_install_deps.erl | 39 ++++---- src/rebar_prv_packages.erl | 12 +-- src/rebar_prv_update.erl | 95 +++++++++---------- src/rebar_state.erl | 52 ++-------- test/mock_pkg_resource.erl | 55 +++++------ test/rebar_pkg_SUITE.erl | 26 +++-- 11 files changed, 220 insertions(+), 265 deletions(-) diff --git a/src/rebar.hrl b/src/rebar.hrl index 8a702dfc..961b8ead 100644 --- a/src/rebar.hrl +++ b/src/rebar.hrl @@ -23,8 +23,14 @@ -define(DEFAULT_RELEASE_DIR, "rel"). -define(DEFAULT_CONFIG_FILE, "rebar.config"). -define(DEFAULT_CDN, "https://s3.amazonaws.com/s3.hex.pm/tarballs"). +-define(DEFAULT_HEX_REGISTRY, "https://s3.amazonaws.com/s3.hex.pm/registry.ets.gz"). -define(LOCK_FILE, "rebar.lock"). +-define(PACKAGE_INDEX_VERSION, 3). +-define(PACKAGE_TABLE, package_index). +-define(INDEX_FILE, "packages.idx"). +-define(REGISTRY_FILE, "registry"). + -ifdef(namespaced_types). -type rebar_dict() :: dict:dict(). -else. diff --git a/src/rebar3.erl b/src/rebar3.erl index 8004443a..ab3e0eb5 100644 --- a/src/rebar3.erl +++ b/src/rebar3.erl @@ -105,6 +105,7 @@ run_aux(State, RawArgs) -> {ok, Providers} = application:get_env(rebar, providers), %% Providers can modify profiles stored in opts, so set default after initializing providers State4 = rebar_state:create_logic_providers(Providers, State3), + rebar_packages:packages(State4), State5 = rebar_plugins:project_apps_install(State4), State6 = rebar_state:default(State5, rebar_state:opts(State5)), @@ -280,9 +281,15 @@ state_from_global_config(Config, GlobalConfigFile) -> %% We don't want to worry about global plugin install state effecting later %% usage. So we throw away the global profile state used for plugin install. GlobalConfigThrowAway = rebar_state:current_profiles(GlobalConfig, [global]), - GlobalState = rebar_plugins:handle_plugins(global, - rebar_state:get(GlobalConfigThrowAway, plugins, []), - GlobalConfigThrowAway), + GlobalState = case rebar_state:get(GlobalConfigThrowAway, plugins, []) of + [] -> + GlobalConfigThrowAway; + GlobalPluginsToInstall -> + rebar_packages:packages(GlobalConfigThrowAway), + rebar_plugins:handle_plugins(global, + GlobalPluginsToInstall, + GlobalConfigThrowAway) + end, GlobalPlugins = rebar_state:providers(GlobalState), GlobalConfig2 = rebar_state:set(GlobalConfig, plugins, []), GlobalConfig3 = rebar_state:set(GlobalConfig2, {plugins, global}, rebar_state:get(GlobalConfigThrowAway, plugins, [])), diff --git a/src/rebar_app_utils.erl b/src/rebar_app_utils.erl index 17b435c5..b14c50af 100644 --- a/src/rebar_app_utils.erl +++ b/src/rebar_app_utils.erl @@ -212,13 +212,8 @@ parse_goal(Name, Constraint) -> end. get_package(Dep, State) -> - case rebar_state:registry(State) of - {ok, T} -> - {ok, HighestDepVsn} = rebar_packages:find_highest_matching(Dep, "0", T), - {Dep, HighestDepVsn}; - error -> - throw(?PRV_ERROR({load_registry_fail, Dep})) - end. + {ok, HighestDepVsn} = rebar_packages:find_highest_matching(Dep, "0", ?PACKAGE_TABLE, State), + {Dep, HighestDepVsn}. -spec has_all_beams(file:filename_all(), [module()]) -> true | ?PRV_ERROR({missing_module, module()}). diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl index 5f3d1c90..f6993c56 100644 --- a/src/rebar_packages.erl +++ b/src/rebar_packages.erl @@ -1,71 +1,60 @@ -module(rebar_packages). -export([packages/1 - ,packages_graph/1 - ,registry/1 + ,close_packages/0 + ,load_and_verify_version/1 + ,deps/3 ,package_dir/1 - ,check_registry/3 ,registry_checksum/2 - ,find_highest_matching/3]). + ,find_highest_matching/4 + ,format_error/1]). -export_type([package/0]). -include("rebar.hrl"). +-include_lib("providers/include/providers.hrl"). -type pkg_name() :: binary() | atom(). -type vsn() :: binary(). -type package() :: pkg_name() | {pkg_name(), vsn()}. --spec packages(rebar_state:t()) -> rebar_dict(). -%% DON'T USE IT! Use rebar_state:packages(State) instead. +-spec packages(rebar_state:t()) -> ets:tid(). packages(State) -> - RegistryDir = package_dir(State), - DictFile = filename:join(RegistryDir, "dict"), - - try - {ok, DictBinary} = file:read_file(DictFile), - binary_to_term(DictBinary) - catch - _:_ -> + catch ets:delete(?PACKAGE_TABLE), + case load_and_verify_version(State) of + true -> + ok; + false -> + ?DEBUG("Error loading package index.", []), ?ERROR("Bad packages index, try to fix with `rebar3 update`", []), - dict:new() + ets:new(?PACKAGE_TABLE, [named_table, public]) end. --spec packages_graph(rebar_state:t()) -> rebar_digraph(). -packages_graph(State) -> - RegistryDir = package_dir(State), - Edges = filename:join(RegistryDir, "edges"), - Vertices = filename:join(RegistryDir, "vertices"), - Neighbors = filename:join(RegistryDir, "neighbors"), +close_packages() -> + catch ets:delete(?PACKAGE_TABLE). - case lists:all(fun(X) -> filelib:is_file(X) end, [Edges, Vertices, Neighbors]) of - true -> - try - {ok, EdgesTab} = ets:file2tab(Edges), - {ok, VerticesTab} = ets:file2tab(Vertices), - {ok, NeighborsTab} = ets:file2tab(Neighbors), - {digraph, EdgesTab, VerticesTab, NeighborsTab, true} - catch - _:_ -> - ?ERROR("Bad packages index, try to fix with `rebar3 update`", []), - digraph:new() +load_and_verify_version(State) -> + RegistryDir = package_dir(State), + case ets:file2tab(filename:join(RegistryDir, ?INDEX_FILE)) of + {ok, _} -> + case ets:lookup_element(?PACKAGE_TABLE, package_index_version, 2) of + ?PACKAGE_INDEX_VERSION -> + true; + _ -> + (catch ets:delete(?PACKAGE_TABLE)), + rebar_prv_update:hex_to_index(State) end; - false -> - ?ERROR("Bad packages index, try to fix with `rebar3 update`", []), - digraph:new() + _ -> + rebar_prv_update:hex_to_index(State) end. --spec registry(rebar_state:t()) -> {ok, ets:tid()} | {error, any()}. -%% DON'T USE IT! Use rebar_state:registry(State) instead. -registry(State) -> - RegistryDir = package_dir(State), - HexFile = filename:join(RegistryDir, "registry"), - case ets:file2tab(HexFile) of - {ok, T} -> - {ok, T}; - {error, Reason} -> - ?DEBUG("Error loading registry: ~p", [Reason]), - error +deps(Name, Vsn, State) -> + try + verify_table(State), + ets:lookup_element(?PACKAGE_TABLE, {ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)}, 2) + catch + _:_ -> + throw(?PRV_ERROR({missing_package, ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)})) end. package_dir(State) -> @@ -78,27 +67,13 @@ package_dir(State) -> ok = filelib:ensure_dir(filename:join(PackageDir, "placeholder")), PackageDir. - -check_registry(Pkg, Vsn, State) -> - case rebar_state:registry(State) of - {ok, T} -> - case ets:lookup(T, Pkg) of - [{Pkg, [Vsns]}] -> - lists:member(Vsn, Vsns); - _ -> - false - end; - error -> - false - end. - registry_checksum({pkg, Name, Vsn}, State) -> - {ok, Registry} = rebar_state:registry(State), - case ets:lookup(Registry, {Name, Vsn}) of - [{{_, _}, [_, Checksum | _]}] -> - Checksum; - [] -> - none + try + verify_table(State), + ets:lookup_element(?PACKAGE_TABLE, {Name, Vsn}, 3) + catch + _:_ -> + throw(?PRV_ERROR({missing_package, ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)})) end. %% Hex supports use of ~> to specify the version required for a dependency. @@ -116,28 +91,45 @@ registry_checksum({pkg, Name, Vsn}, State) -> %% `~> 2.1.3-dev` | `>= 2.1.3-dev and < 2.2.0` %% `~> 2.0` | `>= 2.0.0 and < 3.0.0` %% `~> 2.1` | `>= 2.1.0 and < 3.0.0` -find_highest_matching(Dep, Constraint, T) -> - case ets:lookup(T, Dep) of - [{Dep, [[Vsn]]}] -> - case ec_semver:pes(Vsn, Constraint) of - true -> - {ok, Vsn}; - false -> - ?WARN("Only existing version of ~s is ~s which does not match constraint ~~> ~s. " - "Using anyway, but it is not guarenteed to work.", [Dep, Vsn, Constraint]), - {ok, Vsn} - end; - [{Dep, [[HeadVsn | VsnTail]]}] -> - {ok, lists:foldl(fun(Version, Highest) -> - case ec_semver:pes(Version, Constraint) andalso - ec_semver:gt(Version, Highest) of - true -> - Version; - false -> - Highest - end - end, HeadVsn, VsnTail)}; +find_highest_matching(Dep, Constraint, Table, State) -> + verify_table(State), + case ets:lookup_element(Table, Dep, 2) of + [[HeadVsn | VsnTail]] -> + {ok, handle_vsns(Constraint, HeadVsn, VsnTail)}; + [[Vsn]] -> + handle_single_vsn(Dep, Vsn, Constraint); + [Vsn] -> + handle_single_vsn(Dep, Vsn, Constraint); + [HeadVsn | VsnTail] -> + {ok, handle_vsns(Constraint, HeadVsn, VsnTail)}; [] -> - ?WARN("Missing registry entry for package ~s", [Dep]), + ?WARN("Missing registry entry for package ~s. Try to fix with `rebar3 update`", [Dep]), none end. + +handle_vsns(Constraint, HeadVsn, VsnTail) -> + lists:foldl(fun(Version, Highest) -> + case ec_semver:pes(Version, Constraint) andalso + ec_semver:gt(Version, Highest) of + true -> + Version; + false -> + Highest + end + end, HeadVsn, VsnTail). + +handle_single_vsn(Dep, Vsn, Constraint) -> + case ec_semver:pes(Vsn, Constraint) of + true -> + {ok, Vsn}; + false -> + ?WARN("Only existing version of ~s is ~s which does not match constraint ~~> ~s. " + "Using anyway, but it is not guarenteed to work.", [Dep, Vsn, Constraint]), + {ok, Vsn} + end. + +format_error({missing_package, Package, Version}) -> + io_lib:format("Package not found in registry: ~s-~s. Try to fix with `rebar3 update`", [Package, Version]). + +verify_table(State) -> + ets:info(?PACKAGE_TABLE, named_table) =:= true orelse ?MODULE:load_and_verify_version(State). diff --git a/src/rebar_plugins.erl b/src/rebar_plugins.erl index bda3fb7c..742dc828 100644 --- a/src/rebar_plugins.erl +++ b/src/rebar_plugins.erl @@ -16,10 +16,12 @@ -spec project_apps_install(rebar_state:t()) -> rebar_state:t(). project_apps_install(State) -> + rebar_packages:packages(State), + Profiles = rebar_state:current_profiles(State), ProjectApps = rebar_state:project_apps(State), - lists:foldl(fun(Profile, StateAcc) -> + State1 = lists:foldl(fun(Profile, StateAcc) -> Plugins = rebar_state:get(State, {plugins, Profile}, []), StateAcc1 = handle_plugins(Profile, Plugins, StateAcc), @@ -30,7 +32,11 @@ project_apps_install(State) -> Plugins2 = rebar_state:get(S, {plugins, Profile}, []), handle_plugins(Profile, Plugins2, StateAcc2) end, StateAcc1, ProjectApps) - end, State, Profiles). + end, State, Profiles), + + rebar_packages:close_packages(), + + State1. -spec install(rebar_state:t()) -> rebar_state:t(). install(State) -> diff --git a/src/rebar_prv_install_deps.erl b/src/rebar_prv_install_deps.erl index 0883fabe..0b0f607b 100644 --- a/src/rebar_prv_install_deps.erl +++ b/src/rebar_prv_install_deps.erl @@ -125,10 +125,7 @@ handle_deps_as_profile(Profile, State, Deps, Upgrade) -> DepsDir = profile_dep_dir(State, Profile), Deps1 = rebar_app_utils:parse_deps(DepsDir, Deps, State, Locks, Level), ProfileLevelDeps = [{Profile, Deps1, Level}], - Graph = rebar_state:packages_graph(State), - Registry = rebar_packages:registry(State), - State1 = rebar_state:packages_graph(rebar_state:registry(State, Registry), Graph), - handle_profile_level(ProfileLevelDeps, [], sets:new(), Upgrade, Locks, State1, Graph). + handle_profile_level(ProfileLevelDeps, [], sets:new(), Upgrade, Locks, State). %% =================================================================== %% Internal functions @@ -141,10 +138,7 @@ deps_per_profile(Profiles, Upgrade, State) -> Deps = lists:foldl(fun(Profile, DepAcc) -> [parsed_profile_deps(State, Profile, Level) | DepAcc] end, [], Profiles), - Graph = rebar_state:packages_graph(State), - Registry = rebar_packages:registry(State), - State1 = rebar_state:packages_graph(rebar_state:registry(State, Registry), Graph), - handle_profile_level(Deps, [], sets:new(), Upgrade, Locks, State1, Graph). + handle_profile_level(Deps, [], sets:new(), Upgrade, Locks, State). parsed_profile_deps(State, Profile, Level) -> ParsedDeps = rebar_state:get(State, {parsed_deps, Profile}, []), @@ -153,17 +147,17 @@ parsed_profile_deps(State, Profile, Level) -> %% Level-order traversal of all dependencies, across profiles. %% If profiles x,y,z are present, then the traversal will go: %% x0, y0, z0, x1, y1, z1, ..., xN, yN, zN. -handle_profile_level([], Apps, _Seen, _Upgrade, _Locks, State, _Graph) -> +handle_profile_level([], Apps, _Seen, _Upgrade, _Locks, State) -> {Apps, State}; -handle_profile_level([{Profile, Deps, Level} | Rest], Apps, Seen, Upgrade, Locks, State, Graph) -> +handle_profile_level([{Profile, Deps, Level} | Rest], Apps, Seen, Upgrade, Locks, State) -> {Deps1, Apps1, State1, Seen1} = update_deps(Profile, Level, Deps, Apps - ,State, Upgrade, Seen, Locks, Graph), + ,State, Upgrade, Seen, Locks), Deps2 = case Deps1 of [] -> Rest; _ -> Rest ++ [{Profile, Deps1, Level+1}] end, - handle_profile_level(Deps2, Apps1, sets:union(Seen, Seen1), Upgrade, Locks, State1, Graph). + handle_profile_level(Deps2, Apps1, sets:union(Seen, Seen1), Upgrade, Locks, State1). find_cycles(Apps) -> case rebar_digraph:compile_order(Apps) of @@ -201,17 +195,17 @@ maybe_lock(Profile, AppInfo, Seen, State, Level) -> {sets:add_element(Name, Seen), State} end. -update_deps(Profile, Level, Deps, Apps, State, Upgrade, Seen, Locks, Graph) -> +update_deps(Profile, Level, Deps, Apps, State, Upgrade, Seen, Locks) -> lists:foldl( fun(AppInfo, {DepsAcc, AppsAcc, StateAcc, SeenAcc}) -> update_dep(AppInfo, Profile, Level, DepsAcc, AppsAcc, StateAcc, - Upgrade, SeenAcc, Locks, Graph) + Upgrade, SeenAcc, Locks) end, {[], Apps, State, Seen}, rebar_utils:sort_deps(Deps)). -update_dep(AppInfo, Profile, Level, Deps, Apps, State, Upgrade, Seen, Locks, Graph) -> +update_dep(AppInfo, Profile, Level, Deps, Apps, State, Upgrade, Seen, Locks) -> %% If not seen, add to list of locks to write out Name = rebar_app_info:name(AppInfo), case sets:is_element(Name, Seen) of @@ -222,7 +216,7 @@ update_dep(AppInfo, Profile, Level, Deps, Apps, State, Upgrade, Seen, Locks, Gra false -> update_unseen_dep(AppInfo, Profile, Level, Deps, Apps, - State, Upgrade, Seen, Locks, Graph) + State, Upgrade, Seen, Locks) end. profile_dep_dir(State, Profile) -> @@ -242,24 +236,23 @@ update_seen_dep(AppInfo, _Profile, _Level, Deps, Apps, State, Upgrade, Seen, Loc end, {Deps, Apps, State, Seen}. -update_unseen_dep(AppInfo, Profile, Level, Deps, Apps, State, Upgrade, Seen, Locks, Graph) -> +update_unseen_dep(AppInfo, Profile, Level, Deps, Apps, State, Upgrade, Seen, Locks) -> {NewSeen, State1} = maybe_lock(Profile, AppInfo, Seen, State, Level), {_, AppInfo1} = maybe_fetch(AppInfo, Profile, Upgrade, Seen, State1), DepsDir = profile_dep_dir(State, Profile), {AppInfo2, NewDeps, State2} = - handle_dep(State1, Profile, DepsDir, AppInfo1, Locks, Level, Graph), + handle_dep(State1, Profile, DepsDir, AppInfo1, Locks, Level), AppInfo3 = rebar_app_info:dep_level(AppInfo2, Level), {NewDeps ++ Deps, [AppInfo3 | Apps], State2, NewSeen}. --spec handle_dep(rebar_state:t(), atom(), file:filename_all(), rebar_app_info:t(), list(), integer(), rebar_dict()) -> {rebar_app_info:t(), [rebar_app_info:t()], [pkg_dep()], [integer()], rebar_state:t()}. -handle_dep(State, Profile, DepsDir, AppInfo, Locks, Level, Graph) -> +-spec handle_dep(rebar_state:t(), atom(), file:filename_all(), rebar_app_info:t(), list(), integer()) -> {rebar_app_info:t(), [rebar_app_info:t()], [pkg_dep()], [integer()], rebar_state:t()}. +handle_dep(State, Profile, DepsDir, AppInfo, Locks, Level) -> Profiles = rebar_state:current_profiles(State), Name = rebar_app_info:name(AppInfo), Vsn = rebar_app_info:original_vsn(AppInfo), %% Deps may be under a sub project app, find it and use its state if so - S0 = rebar_app_info:state(AppInfo), - S = rebar_state:registry(S0, rebar_state:registry(State)), + S = rebar_app_info:state(AppInfo), C = rebar_config:consult(rebar_app_info:dir(AppInfo)), S1 = rebar_state:new(S, C, rebar_app_info:dir(AppInfo)), S2 = rebar_state:apply_overrides(S1, Name), @@ -278,7 +271,7 @@ handle_dep(State, Profile, DepsDir, AppInfo, Locks, Level, Graph) -> %% Upgrade lock level to be the level the dep will have in this dep tree case rebar_app_info:resource_type(AppInfo1) of pkg -> - NewDeps = digraph:out_neighbours(Graph, {ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)}), + NewDeps = rebar_packages:deps(Name, Vsn, S5), NewDeps1 = rebar_app_utils:parse_deps(Name, DepsDir, NewDeps, S5, Locks, Level+1), {rebar_app_info:deps(AppInfo1, NewDeps), NewDeps1, State}; _ -> diff --git a/src/rebar_prv_packages.erl b/src/rebar_prv_packages.erl index 44cde4e1..f5d9e38d 100644 --- a/src/rebar_prv_packages.erl +++ b/src/rebar_prv_packages.erl @@ -27,19 +27,17 @@ init(State) -> -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. do(State) -> - Dict = rebar_packages:packages(State), - print_packages(Dict), + print_packages(), {ok, State}. -spec format_error(any()) -> iolist(). format_error(load_registry_fail) -> "Failed to load package regsitry. Try running 'rebar3 update' to fix". -print_packages(Dict) -> - Pkgs = lists:keysort(1, dict:fetch_keys(Dict)), - SortedPkgs = lists:foldl(fun({Pkg, Vsn}, Acc) -> - orddict:append_list(Pkg, [Vsn], Acc) - end, orddict:new(), Pkgs), +print_packages() -> + SortedPkgs = ets:foldl(fun({{Pkg, Vsn}, _}, Acc) -> + orddict:append_list(Pkg, [Vsn], Acc) + end, orddict:new(), ?PACKAGE_TABLE), orddict:map(fun(Name, Vsns) -> SortedVsns = lists:sort(fun(A, B) -> diff --git a/src/rebar_prv_update.erl b/src/rebar_prv_update.erl index e73c7693..4b3a1553 100644 --- a/src/rebar_prv_update.erl +++ b/src/rebar_prv_update.erl @@ -9,21 +9,14 @@ do/1, format_error/1]). +-export([hex_to_index/1]). + -include("rebar.hrl"). -include_lib("providers/include/providers.hrl"). -define(PROVIDER, update). -define(DEPS, []). -%% Ignore warning of digraph opaque type when running dialyzer --dialyzer({no_opaque, do/1}). --dialyzer({no_opaque, write_registry/3}). - -%% Ignoring the opaque type warning won't stop dialyzer from warning of -%% no return for functions that had the opaque type warnings --dialyzer({no_return, do/1}). --dialyzer({no_return, write_registry/3}). - %% =================================================================== %% Public API %% =================================================================== @@ -50,16 +43,15 @@ do(State) -> TmpDir = ec_file:insecure_mkdtemp(), TmpFile = filename:join(TmpDir, "packages.gz"), - Url = rebar_state:get(State, rebar_packages_cdn, "https://s3.amazonaws.com/s3.hex.pm/registry.ets.gz"), + Url = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_HEX_REGISTRY), {ok, _RequestId} = httpc:request(get, {Url, []}, [], [{stream, TmpFile}, {sync, true}], rebar), {ok, Data} = file:read_file(TmpFile), Unzipped = zlib:gunzip(Data), ok = file:write_file(HexFile, Unzipped), - {Dict, Graph} = hex_to_graph(HexFile), - write_registry(Dict, Graph, State), - true = digraph:delete(Graph), + + hex_to_index(State), ok catch _E:C -> @@ -73,57 +65,58 @@ do(State) -> format_error(package_index_write) -> "Failed to write package index.". --spec write_registry(rebar_dict(), {digraph, ets:tid(), ets:tid(), ets:tid(), any()}, rebar_state:t()) -> ok | {error, atom()}. -write_registry(Dict, {digraph, Edges, Vertices, Neighbors, _}, State) -> - RegistryDir = rebar_packages:package_dir(State), - filelib:ensure_dir(filename:join(RegistryDir, "dummy")), - ets:tab2file(Edges, filename:join(RegistryDir, "edges")), - ets:tab2file(Vertices, filename:join(RegistryDir, "vertices")), - ets:tab2file(Neighbors, filename:join(RegistryDir, "neighbors")), - file:write_file(filename:join(RegistryDir, "dict"), term_to_binary(Dict)). - is_supported(<<"make">>) -> true; is_supported(<<"rebar">>) -> true; is_supported(<<"rebar3">>) -> true; is_supported(_) -> false. -hex_to_graph(Filename) -> - {ok, T} = ets:file2tab(Filename), - Graph = digraph:new(), - ets:foldl(fun({Pkg, [Versions]}, ok) when is_binary(Pkg), is_list(Versions) -> - lists:foreach(fun(Version) -> - digraph:add_vertex(Graph, {Pkg, Version}, 1) - end, Versions); - (_, ok) -> - ok - end, ok, T), - - Dict1 = ets:foldl(fun({{Pkg, PkgVsn}, [Deps, _, BuildTools | _]}, Dict) when is_list(BuildTools) -> - case lists:any(fun is_supported/1, BuildTools) of - true -> - DepsList = update_graph(Pkg, PkgVsn, Deps, T, Graph), - dict:store({Pkg, PkgVsn}, DepsList, Dict); - false -> - Dict - end; - (_, Dict) -> - Dict - end, dict:new(), T), - {Dict1, Graph}. - -update_graph(Pkg, PkgVsn, Deps, HexRegistry, Graph) -> +hex_to_index(State) -> + RegistryDir = rebar_packages:package_dir(State), + HexFile = filename:join(RegistryDir, "registry"), + try ets:file2tab(HexFile) of + {ok, Registry} -> + try + (catch ets:delete(?PACKAGE_TABLE)), + ets:new(?PACKAGE_TABLE, [named_table, public]), + ets:foldl(fun({{Pkg, PkgVsn}, [Deps, Checksum, BuildTools | _]}, _) when is_list(BuildTools) -> + case lists:any(fun is_supported/1, BuildTools) of + true -> + DepsList = update_deps_list(Deps, Registry, State), + ets:insert(?PACKAGE_TABLE, {{Pkg, PkgVsn}, DepsList, Checksum}); + false -> + true + end; + ({Pkg, [Vsns]}, _) when is_binary(Pkg) -> + ets:insert(?PACKAGE_TABLE, {Pkg, Vsns}); + (_, _) -> + true + end, true, Registry), + + ets:insert(?PACKAGE_TABLE, {package_index_version, ?PACKAGE_INDEX_VERSION}), + ets:tab2file(?PACKAGE_TABLE, filename:join(RegistryDir, "packages.idx")), + true + after + catch ets:delete(Registry) + end; + {error, Reason} -> + ?DEBUG("Error loading package registry: ~p", [Reason]), + false + catch + _:_ -> + fail + end. + +update_deps_list(Deps, HexRegistry, State) -> lists:foldl(fun([Dep, DepVsn, false, _AppName | _], DepsListAcc) -> case DepVsn of <<"~> ", Vsn/binary>> -> - case rebar_packages:find_highest_matching(Dep, Vsn, HexRegistry) of + case rebar_packages:find_highest_matching(Dep, Vsn, HexRegistry, State) of {ok, HighestDepVsn} -> - digraph:add_edge(Graph, {Pkg, PkgVsn}, {Dep, HighestDepVsn}), - [{Dep, DepVsn} | DepsListAcc]; + [{Dep, HighestDepVsn} | DepsListAcc]; none -> DepsListAcc end; Vsn -> - digraph:add_edge(Graph, {Pkg, PkgVsn}, {Dep, Vsn}), [{Dep, Vsn} | DepsListAcc] end; ([_Dep, _DepVsn, true, _AppName | _], DepsListAcc) -> diff --git a/src/rebar_state.erl b/src/rebar_state.erl index 54e7f131..213eb2f9 100644 --- a/src/rebar_state.erl +++ b/src/rebar_state.erl @@ -38,10 +38,6 @@ overrides/1, overrides/2, apply_overrides/2, - packages_graph/1, packages_graph/2, - packages/1, - registry/1, registry/2, - resources/1, resources/2, add_resource/2, providers/1, providers/2, add_provider/2]). @@ -66,9 +62,6 @@ all_plugin_deps = [] :: [rebar_app_info:t()], all_deps = [] :: [rebar_app_info:t()], - packages = undefined :: rebar_dict(), - packages_graph = undefined :: rebar_digraph() | undefined, - registry = undefined :: {ok, ets:tid()} | error | undefined, overrides = [], resources = [], providers = []}). @@ -90,18 +83,9 @@ new(Config) when is_list(Config) -> Terms = [{{deps, default}, Deps}, {{plugins, default}, Plugins} | Config], true = rebar_config:verify_config_format(Terms), Opts = dict:from_list(Terms), - load_package_registry( - BaseState#state_t { dir = rebar_dir:get_cwd(), - default = Opts, - opts = Opts }). - -load_package_registry(Config0) -> - Registry = rebar_packages:registry(Config0), - Packages = rebar_packages:packages(Config0), - PackagesGraph = rebar_packages:packages_graph(Config0), - Config0#state_t{registry = Registry, - packages_graph = PackagesGraph, - packages = Packages}. + BaseState#state_t { dir = rebar_dir:get_cwd(), + default = Opts, + opts = Opts }. -spec new(t() | atom(), list()) -> t(). new(Profile, Config) when is_atom(Profile) @@ -113,11 +97,10 @@ new(Profile, Config) when is_atom(Profile) Terms = [{{deps, default}, Deps}, {{plugins, default}, Plugins} | Config], true = rebar_config:verify_config_format(Terms), Opts = dict:from_list(Terms), - load_package_registry( - BaseState#state_t { dir = rebar_dir:get_cwd(), - current_profiles = [Profile], - default = Opts, - opts = Opts }); + BaseState#state_t { dir = rebar_dir:get_cwd(), + current_profiles = [Profile], + default = Opts, + opts = Opts }; new(ParentState=#state_t{}, Config) -> %% Load terms from rebar.config, if it exists Dir = rebar_dir:get_cwd(), @@ -450,27 +433,6 @@ namespace(#state_t{namespace=Namespace}) -> namespace(State=#state_t{}, Namespace) -> State#state_t{namespace=Namespace}. -packages(#state_t{packages=undefined}) -> - throw(packages_usage_error); -packages(#state_t{packages=Packages}) -> - Packages. - -packages_graph(#state_t{packages_graph=undefined}) -> - throw(packages_usage_error); -packages_graph(#state_t{packages_graph=PackagesGraph}) -> - PackagesGraph. - -packages_graph(State, PackagesGraph) -> - State#state_t{packages_graph=PackagesGraph}. - -registry(#state_t{registry=undefined}) -> - throw(registry_usage_error); -registry(#state_t{registry=Registry}) -> - Registry. - -registry(State, Registry) -> - State#state_t{registry=Registry}. - -spec resources(t()) -> [{rebar_resource:type(), module()}]. resources(#state_t{resources=Resources}) -> Resources. diff --git a/test/mock_pkg_resource.erl b/test/mock_pkg_resource.erl index 9258f72d..4ed8d8ea 100644 --- a/test/mock_pkg_resource.erl +++ b/test/mock_pkg_resource.erl @@ -35,7 +35,7 @@ mock(Opts) -> unmock() -> meck:unload(?MOD), - meck:unload(rebar_state). + meck:unload(rebar_packages). %%%%%%%%%%%%%%% %%% Private %%% @@ -111,34 +111,19 @@ mock_pkg_index(Opts) -> Deps = proplists:get_value(pkgdeps, Opts, []), Skip = proplists:get_value(not_in_index, Opts, []), %% Dict: {App, Vsn}: [{<<"link">>, <<>>}, {<<"deps">>, []}] - %% Digraph: all apps and deps in the index + %% Index: all apps and deps in the index + Dict = find_parts(Deps, Skip), - GraphParts = to_graph_parts(Dict), - Digraph = rebar_digraph:restore_graph(GraphParts), - meck:new(rebar_state, [passthrough, no_link]), - meck:expect(rebar_state, registry, - fun(_State) -> {ok, to_registry(Deps)} end), - meck:expect(rebar_state, packages, - fun(_State) -> Dict end), - meck:expect(rebar_state, packages_graph, - fun(_State) -> Digraph end). + meck:new(rebar_packages, [passthrough, no_link]), + meck:expect(rebar_packages, packages, + fun(_State) -> to_index(Deps, Dict) end), + meck:expect(rebar_packages, load_and_verify_version, + fun(_State) -> to_index(Deps, Dict), true end). %%%%%%%%%%%%%%% %%% Helpers %%% %%%%%%%%%%%%%%% -to_registry(Deps) -> - Tid = ets:new(registry, []), - lists:foreach(fun({{Name, Vsn}, _}) -> - case ets:lookup(Tid, Name) of - [{_, [Vsns]}] -> - ets:insert(Tid, {Name, [[Vsn | Vsns]]}); - _ -> - ets:insert(Tid, {Name, [[Vsn]]}) - end - end, Deps), - Tid. - all_files(Dir) -> filelib:wildcard(filename:join([Dir, "**"])). @@ -158,10 +143,20 @@ find_parts([{AppName, Deps}|Rest], Skip, Acc) -> find_parts(Rest, Skip, AccNew) end. -to_graph_parts(Dict) -> - LastUpdated = os:timestamp(), - dict:fold(fun(K,Deps,{Ks,Vs}) -> - {[{K,LastUpdated}|Ks], - [{K,{list_to_binary(atom_to_list(DK)), list_to_binary(DV)}} - || {DK,DV} <- Deps] ++ Vs} - end, {[],[]}, Dict). +to_index(AllDeps, Dict) -> + catch ets:delete(package_index), + ets:new(package_index, [named_table, public]), + dict:fold( + fun(K, Deps, _) -> + DepsList = [{ec_cnv:to_binary(DK), ec_cnv:to_binary(DV)} || {DK, DV} <- Deps], + ets:insert(package_index, {K, DepsList, <<"checksum">>}) + end, ok, Dict), + ets:insert(package_index, {package_index_version, 3}), + lists:foreach(fun({{Name, Vsn}, _}) -> + case ets:lookup(package_index, ec_cnv:to_binary(Name)) of + [{_, Vsns}] -> + ets:insert(package_index, {ec_cnv:to_binary(Name), [ec_cnv:to_binary(Vsn) | Vsns]}); + _ -> + ets:insert(package_index, {ec_cnv:to_binary(Name), [ec_cnv:to_binary(Vsn)]}) + end + end, AllDeps). diff --git a/test/rebar_pkg_SUITE.erl b/test/rebar_pkg_SUITE.erl index e99e7878..3cd3a672 100644 --- a/test/rebar_pkg_SUITE.erl +++ b/test/rebar_pkg_SUITE.erl @@ -159,25 +159,33 @@ mock_config(Name, Config) -> Priv = ?config(priv_dir, Config), CacheRoot = filename:join([Priv, "cache", atom_to_list(Name)]), TmpDir = filename:join([Priv, "tmp", atom_to_list(Name)]), - T = ets:new(fake_registry, [public]), - ets:insert_new(T, [ - {{<<"badindexchk">>,<<"1.0.0">>}, [[], ?bad_checksum]}, - {{<<"goodpkg">>,<<"1.0.0">>}, [[], ?good_checksum]}, - {{<<"badpkg">>,<<"1.0.0">>}, [[], ?good_checksum]} + Tid = ets:new(registry_table, [public]), + ets:insert_new(Tid, [ + {<<"badindexchk">>,[[<<"1.0.0">>]]}, + {<<"goodpkg">>,[[<<"1.0.0">>]]}, + {<<"badpkg">>,[[<<"1.0.0">>]]}, + {{<<"badindexchk">>,<<"1.0.0">>}, [[], ?bad_checksum, [<<"rebar3">>]]}, + {{<<"goodpkg">>,<<"1.0.0">>}, [[], ?good_checksum, [<<"rebar3">>]]}, + {{<<"badpkg">>,<<"1.0.0">>}, [[], ?good_checksum, [<<"rebar3">>]]} ]), CacheDir = filename:join([CacheRoot, "hex", "com", "test", "packages"]), filelib:ensure_dir(filename:join([CacheDir, "registry"])), - ok = ets:tab2file(T, filename:join([CacheDir, "registry"])), + ok = ets:tab2file(Tid, filename:join([CacheDir, "registry"])), + %% The state returns us a fake registry meck:new(rebar_state, [passthrough]), - meck:expect(rebar_state, registry, - fun(_State) -> {ok, T} end), meck:expect(rebar_state, get, fun(_State, rebar_packages_cdn, _Default) -> "http://test.com/" end), + meck:new(rebar_dir, [passthrough]), meck:expect(rebar_dir, global_cache_dir, fun(_) -> CacheRoot end), + + meck:new(rebar_packages, [passthrough]), + meck:expect(rebar_packages, package_dir, fun(_) -> CacheDir end), + rebar_prv_update:hex_to_index(rebar_state:new()), + %% Cache fetches are mocked -- we assume the server and clients are %% correctly used. GoodCache = ?config(good_cache, Config), @@ -194,7 +202,7 @@ mock_config(Name, Config) -> [{cache_root, CacheRoot}, {cache_dir, CacheDir}, {tmp_dir, TmpDir}, - {mock_table, T} | Config]. + {mock_table, Tid} | Config]. unmock_config(Config) -> meck:unload(), From e336d7aaa124bb63c92f4470429d06f5de8e089a Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Fri, 21 Aug 2015 19:48:26 -0500 Subject: [PATCH 4/5] remove unneeded packages ets table load/unload in plugins --- src/rebar_plugins.erl | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/src/rebar_plugins.erl b/src/rebar_plugins.erl index 742dc828..6c2daef5 100644 --- a/src/rebar_plugins.erl +++ b/src/rebar_plugins.erl @@ -16,27 +16,21 @@ -spec project_apps_install(rebar_state:t()) -> rebar_state:t(). project_apps_install(State) -> - rebar_packages:packages(State), - Profiles = rebar_state:current_profiles(State), ProjectApps = rebar_state:project_apps(State), - State1 = lists:foldl(fun(Profile, StateAcc) -> + lists:foldl(fun(Profile, StateAcc) -> Plugins = rebar_state:get(State, {plugins, Profile}, []), StateAcc1 = handle_plugins(Profile, Plugins, StateAcc), lists:foldl(fun(App, StateAcc2) -> - AppDir = rebar_app_info:dir(App), - C = rebar_config:consult(AppDir), - S = rebar_state:new(rebar_state:new(), C, AppDir), - Plugins2 = rebar_state:get(S, {plugins, Profile}, []), - handle_plugins(Profile, Plugins2, StateAcc2) - end, StateAcc1, ProjectApps) - end, State, Profiles), - - rebar_packages:close_packages(), - - State1. + AppDir = rebar_app_info:dir(App), + C = rebar_config:consult(AppDir), + S = rebar_state:new(rebar_state:new(), C, AppDir), + Plugins2 = rebar_state:get(S, {plugins, Profile}, []), + handle_plugins(Profile, Plugins2, StateAcc2) + end, StateAcc1, ProjectApps) + end, State, Profiles). -spec install(rebar_state:t()) -> rebar_state:t(). install(State) -> From e853e12d7c5b3f0f29670d4ae89d17d004edada1 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Fri, 21 Aug 2015 21:15:25 -0500 Subject: [PATCH 5/5] add back removed tests --- test/rebar_install_deps_SUITE.erl | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/test/rebar_install_deps_SUITE.erl b/test/rebar_install_deps_SUITE.erl index 9d536d11..b8b70b33 100644 --- a/test/rebar_install_deps_SUITE.erl +++ b/test/rebar_install_deps_SUITE.erl @@ -17,7 +17,7 @@ groups() -> {mixed, [], [ m_flat1, m_flat2, m_circular1, m_circular2, m_pick_source1, m_pick_source2, m_pick_source3, - m_source_to_pkg, + m_pick_source4, m_pick_source5, m_source_to_pkg, m_pkg_level1, m_pkg_level2, m_pkg_level3, m_pkg_level3_alpha_order ]} ]. @@ -96,9 +96,9 @@ format_expected_mdeps(Deps) -> format_expected_mixed_warnings(Warnings) -> [case W of {N, Vsn} when hd(N) >= $a, hd(N) =< $z -> {pkg, string:to_upper(N), Vsn}; - {N, Vsn} when hd(N) >= $A, hd(N) =< $Z -> {src, N, Vsn}; + {N, Vsn} when hd(N) >= $A, hd(N) =< $Z -> {git, N, Vsn}; N when hd(N) >= $a, hd(N) =< $z -> {pkg, string:to_upper(N), "0.0.0"}; - N when hd(N) >= $A, hd(N) =< $Z -> {src, N, "0.0.0"} + N when hd(N) >= $A, hd(N) =< $Z -> {git, N, "0.0.0"} end || W <- Warnings]. %% format: @@ -215,6 +215,16 @@ mdeps(m_pick_source3) -> {"b", []}], [], {ok, ["B"]}}; +mdeps(m_pick_source4) -> + {[{"b", [{"d", "1", []}]}, + {"C", [{"D", "1", []}]}], + [{"D", "1"}], + {ok, ["b", "C", {"d", "1"}]}}; +mdeps(m_pick_source5) -> + {[{"B", [{"d", "1", []}]}, + {"C", [{"D", "1", []}]}], + [{"D", "1"}], + {ok, ["B", "C", {"d", "1"}]}}; mdeps(m_source_to_pkg) -> {[{"B", [{"c",[{"d", []}]}]}], [], @@ -424,6 +434,8 @@ m_circular2(Config) -> run(Config). m_pick_source1(Config) -> run(Config). m_pick_source2(Config) -> run(Config). m_pick_source3(Config) -> run(Config). +m_pick_source4(Config) -> run(Config). +m_pick_source5(Config) -> run(Config). m_source_to_pkg(Config) -> run(Config). m_pkg_level1(Config) -> run(Config). m_pkg_level2(Config) -> run(Config).