diff --git a/bootstrap b/bootstrap index 4196ed53..fc6da254 100755 --- a/bootstrap +++ b/bootstrap @@ -174,7 +174,7 @@ bootstrap_rebar3() -> Res = symlink_or_copy(filename:absname("src"), filename:absname("_build/default/lib/rebar/src")), true = Res == ok orelse Res == exists, - Sources = ["src/rebar_resource.erl" | filelib:wildcard("src/*.erl")], + Sources = ["src/rebar_resource_v2.erl", "src/rebar_resource.erl" | filelib:wildcard("src/*.erl")], [compile_file(X, [{outdir, "_build/default/lib/rebar/ebin/"} ,return | additional_defines()]) || X <- Sources], code:add_patha(filename:absname("_build/default/lib/rebar/ebin")). diff --git a/src/rebar.hrl b/src/rebar.hrl index 4ea472a4..572cbe88 100644 --- a/src/rebar.hrl +++ b/src/rebar.hrl @@ -33,6 +33,10 @@ -define(HEX_AUTH_FILE, "hex.config"). -define(PUBLIC_HEX_REPO, <<"hexpm">>). +%% ignore this function in all modules +%% not every module that exports it and relies on it being called implements provider +-ignore_xref([{format_error, 1}]). + %% the package record is used in a select match spec which upsets dialyzer %% this is the suggested workaround from Tobias %% http://erlang.org/pipermail/erlang-questions/2009-February/041445.html @@ -46,6 +50,11 @@ dependencies :: [#{package => unicode:unicode_binary(), requirement => unicode:unicode_binary()}] | ms_field()}). +-record(resource, {type :: atom(), + module :: module(), + state :: term(), + implementation :: rebar_resource | rebar_resource_v2}). + -ifdef(namespaced_types). -type rebar_dict() :: dict:dict(). -else. diff --git a/src/rebar3.erl b/src/rebar3.erl index 4fa51629..e87cb196 100644 --- a/src/rebar3.erl +++ b/src/rebar3.erl @@ -194,7 +194,7 @@ init_config() -> ?DEBUG("Load global config file ~ts", [GlobalConfigFile]), try state_from_global_config(Config1, GlobalConfigFile) catch - _:_-> + _:_ -> ?WARN("Global config ~ts exists but can not be read. Ignoring global config values.", [GlobalConfigFile]), rebar_state:new(Config1) end; diff --git a/src/rebar_app_discover.erl b/src/rebar_app_discover.erl index 382b36bb..c02cf073 100644 --- a/src/rebar_app_discover.erl +++ b/src/rebar_app_discover.erl @@ -9,8 +9,7 @@ find_apps/2, find_apps/3, find_app/2, - find_app/3, - find_app/4]). + find_app/3]). -include("rebar.hrl"). -include_lib("providers/include/providers.hrl"). @@ -95,7 +94,7 @@ format_error({missing_module, Module}) -> merge_deps(AppInfo, State) -> %% These steps make sure that hooks and artifacts are run in the context of %% the application they are defined at. If an umbrella structure is used and - %% they are deifned at the top level they will instead run in the context of + %% they are defined at the top level they will instead run in the context of %% the State and at the top level, not as part of an application. CurrentProfiles = rebar_state:current_profiles(State), Default = reset_hooks(rebar_state:default(State), CurrentProfiles), @@ -205,7 +204,7 @@ reset_hooks(Opts, CurrentProfiles) -> -spec all_app_dirs([file:name()]) -> [{file:name(), [file:name()]}]. all_app_dirs(LibDirs) -> lists:flatmap(fun(LibDir) -> - SrcDirs = find_config_src(LibDir, ["src"]), + {_, SrcDirs} = find_config_src(LibDir, ["src"]), app_dirs(LibDir, SrcDirs) end, LibDirs). @@ -278,8 +277,9 @@ find_apps(LibDirs, SrcDirs, Validate) -> %% app info record. -spec find_app(file:filename_all(), valid | invalid | all) -> {true, rebar_app_info:t()} | false. find_app(AppDir, Validate) -> - SrcDirs = find_config_src(AppDir, ["src"]), - find_app(rebar_app_info:new(), AppDir, SrcDirs, Validate). + {Config, SrcDirs} = find_config_src(AppDir, ["src"]), + AppInfo = rebar_app_info:update_opts(rebar_app_info:new(), dict:new(), Config), + find_app_(AppInfo, AppDir, SrcDirs, Validate). %% @doc check that a given app in a directory is there, and whether it's %% valid or not based on the second argument. Returns the related @@ -291,7 +291,7 @@ find_app(AppInfo, AppDir, Validate) -> %% of src/ AppOpts = rebar_app_info:opts(AppInfo), SrcDirs = rebar_dir:src_dirs(AppOpts, ["src"]), - find_app(AppInfo, AppDir, SrcDirs, Validate). + find_app_(AppInfo, AppDir, SrcDirs, Validate). %% @doc check that a given app in a directory is there, and whether it's %% valid or not based on the second argument. The third argument includes @@ -301,6 +301,14 @@ find_app(AppInfo, AppDir, Validate) -> [file:filename_all()], valid | invalid | all) -> {true, rebar_app_info:t()} | false. find_app(AppInfo, AppDir, SrcDirs, Validate) -> + Config = rebar_config:consult(AppDir), + AppInfo1 = rebar_app_info:update_opts(AppInfo, rebar_app_info:opts(AppInfo), Config), + find_app_(AppInfo1, AppDir, SrcDirs, Validate). + +-spec find_app_(rebar_app_info:t(), file:filename_all(), + [file:filename_all()], valid | invalid | all) -> + {true, rebar_app_info:t()} | false. +find_app_(AppInfo, AppDir, SrcDirs, Validate) -> AppFile = filelib:wildcard(filename:join([AppDir, "ebin", "*.app"])), AppSrcFile = lists:append( [filelib:wildcard(filename:join([AppDir, SrcDir, "*.app.src"])) @@ -331,17 +339,14 @@ create_app_info(AppInfo, AppDir, AppFile) -> AppInfo2 = rebar_app_info:applications( rebar_app_info:app_details(AppInfo1, AppDetails), IncludedApplications++Applications), - C = rebar_config:consult(AppDir), - AppInfo3 = rebar_app_info:update_opts(AppInfo2, - rebar_app_info:opts(AppInfo2), C), - Valid = case rebar_app_utils:validate_application_info(AppInfo3) =:= true - andalso rebar_app_info:has_all_artifacts(AppInfo3) =:= true of + Valid = case rebar_app_utils:validate_application_info(AppInfo2) =:= true + andalso rebar_app_info:has_all_artifacts(AppInfo2) =:= true of true -> true; _ -> false end, - rebar_app_info:dir(rebar_app_info:valid(AppInfo3, Valid), AppDir). + rebar_app_info:dir(rebar_app_info:valid(AppInfo2, Valid), AppDir). %% @doc Read in and parse the .app file if it is availabe. Do the same for %% the .app.src file if it exists. @@ -408,7 +413,7 @@ try_handle_app_src_file(_AppInfo, _, _AppDir, [], _Validate) -> try_handle_app_src_file(_AppInfo, _, _AppDir, _AppSrcFile, valid) -> false; try_handle_app_src_file(AppInfo, _, AppDir, [File], Validate) when Validate =:= invalid - ; Validate =:= all -> + ; Validate =:= all -> AppInfo1 = rebar_app_info:app_file(AppInfo, undefined), AppInfo2 = create_app_info(AppInfo1, AppDir, File), case filename:extension(File) of @@ -437,8 +442,8 @@ to_atom(Bin) -> find_config_src(AppDir, Default) -> case rebar_config:consult(AppDir) of [] -> - Default; + {[], Default}; Terms -> %% TODO: handle profiles I guess, but we don't have that info - proplists:get_value(src_dirs, Terms, Default) + {Terms, proplists:get_value(src_dirs, Terms, Default)} end. diff --git a/src/rebar_app_info.erl b/src/rebar_app_info.erl index bbbce44e..162d8e78 100644 --- a/src/rebar_app_info.erl +++ b/src/rebar_app_info.erl @@ -7,6 +7,7 @@ new/4, new/5, update_opts/3, + update_opts_deps/2, discover/1, name/1, name/2, @@ -53,6 +54,8 @@ is_checkout/2, valid/1, valid/2, + is_available/1, + is_available/2, verify_otp_vsn/1, has_all_artifacts/1, @@ -87,7 +90,8 @@ source :: string() | tuple() | checkout | undefined, is_lock=false :: boolean(), is_checkout=false :: boolean(), - valid :: boolean() | undefined}). + valid :: boolean() | undefined, + is_available=false :: boolean()}). %%============================================================================ %% types @@ -152,8 +156,10 @@ new(Parent, AppName, Vsn, Dir, Deps) -> update_opts(AppInfo, Opts, Config) -> LockDeps = case resource_type(AppInfo) of pkg -> - Deps = deps(AppInfo), - [{{locks, default}, Deps}, {{deps, default}, Deps}]; + %% Deps are set separate for packages + %% instead of making it seem we have no deps + %% don't set anything here. + []; _ -> deps_from_config(dir(AppInfo), Config) end, @@ -165,8 +171,18 @@ update_opts(AppInfo, Opts, Config) -> NewOpts = rebar_opts:merge_opts(LocalOpts, Opts), - AppInfo#app_info_t{opts=NewOpts - ,default=NewOpts}. + AppInfo#app_info_t{opts=NewOpts, + default=NewOpts}. + +%% @doc update the opts based on new deps, usually from an app's hex registry metadata +-spec update_opts_deps(t(), [any()]) -> t(). +update_opts_deps(AppInfo=#app_info_t{opts=Opts}, Deps) -> + LocalOpts = dict:from_list([{{locks, default}, Deps}, {{deps, default}, Deps}]), + NewOpts = rebar_opts:merge_opts(LocalOpts, Opts), + AppInfo#app_info_t{opts=NewOpts, + default=NewOpts, + deps=Deps}. + %% @private extract the deps for an app in `Dir' based on its config file data -spec deps_from_config(file:filename(), [any()]) -> [{tuple(), any()}, ...]. @@ -478,6 +494,17 @@ is_checkout(#app_info_t{is_checkout=IsCheckout}) -> is_checkout(AppInfo=#app_info_t{}, IsCheckout) -> AppInfo#app_info_t{is_checkout=IsCheckout}. +%% @doc returns whether the app source exists in the deps dir +-spec is_available(t()) -> boolean(). +is_available(#app_info_t{is_available=IsAvailable}) -> + IsAvailable. + +%% @doc sets whether the app's source is available +%% only set if the app's source is found in the expected dep directory +-spec is_available(t(), boolean()) -> t(). +is_available(AppInfo=#app_info_t{}, IsAvailable) -> + AppInfo#app_info_t{is_available=IsAvailable}. + %% @doc returns whether the app is valid (built) or not -spec valid(t()) -> boolean(). valid(AppInfo=#app_info_t{valid=undefined}) -> diff --git a/src/rebar_app_utils.erl b/src/rebar_app_utils.erl index 2190a90a..4851de42 100644 --- a/src/rebar_app_utils.erl +++ b/src/rebar_app_utils.erl @@ -217,26 +217,23 @@ parse_dep(_, Dep, _, _, _) -> dep_to_app(Parent, DepsDir, Name, Vsn, Source, IsLock, State) -> CheckoutsDir = rebar_utils:to_list(rebar_dir:checkouts_dir(State, Name)), AppInfo = case rebar_app_info:discover(CheckoutsDir) of - {ok, App} -> - rebar_app_info:source(rebar_app_info:is_checkout(App, true), checkout); - not_found -> - Dir = rebar_utils:to_list(filename:join(DepsDir, Name)), - {ok, AppInfo0} = - case rebar_app_info:discover(Dir) of - {ok, App} -> - {ok, rebar_app_info:parent(App, Parent)}; - not_found -> - rebar_app_info:new(Parent, Name, Vsn, Dir, []) - end, - rebar_app_info:source(AppInfo0, Source) - end, - C = rebar_config:consult(rebar_app_info:dir(AppInfo)), - AppInfo1 = rebar_app_info:update_opts(AppInfo, rebar_app_info:opts(AppInfo), C), - Overrides = rebar_state:get(State, overrides, []), - AppInfo2 = rebar_app_info:set(AppInfo1, overrides, rebar_app_info:get(AppInfo, overrides, [])++Overrides), - AppInfo3 = rebar_app_info:apply_overrides(rebar_app_info:get(AppInfo2, overrides, []), AppInfo2), - AppInfo4 = rebar_app_info:apply_profiles(AppInfo3, [default, prod]), - AppInfo5 = rebar_app_info:profiles(AppInfo4, [default]), + {ok, App} -> + rebar_app_info:source(rebar_app_info:is_checkout(App, true), checkout); + not_found -> + Dir = rebar_utils:to_list(filename:join(DepsDir, Name)), + {ok, AppInfo0} = + case rebar_app_info:discover(Dir) of + {ok, App} -> + {ok, rebar_app_info:is_available(rebar_app_info:parent(App, Parent), + true)}; + not_found -> + rebar_app_info:new(Parent, Name, Vsn, Dir, []) + end, + rebar_app_info:source(AppInfo0, Source) + end, + Overrides = rebar_app_info:get(AppInfo, overrides, []) ++ rebar_state:get(State, overrides, []), + AppInfo2 = rebar_app_info:set(AppInfo, overrides, Overrides), + AppInfo5 = rebar_app_info:profiles(AppInfo2, [default]), rebar_app_info:is_lock(AppInfo5, IsLock). %% @doc Takes a given application app_info record along with the project. @@ -250,7 +247,7 @@ expand_deps_sources(Dep, State) -> %% around version if required. -spec update_source(rebar_app_info:t(), Source, rebar_state:t()) -> rebar_app_info:t() when - Source :: rebar_resource:source(). + Source :: rebar_resource_v2:source(). update_source(AppInfo, {pkg, PkgName, PkgVsn, Hash}, State) -> case rebar_packages:resolve_version(PkgName, PkgVsn, Hash, ?PACKAGE_TABLE, State) of @@ -259,8 +256,12 @@ update_source(AppInfo, {pkg, PkgName, PkgVsn, Hash}, State) -> checksum = Hash1, dependencies = Deps} = Package, AppInfo1 = rebar_app_info:source(AppInfo, {pkg, PkgName, PkgVsn1, Hash1, RepoConfig}), - AppInfo2 = rebar_app_info:resource_type(rebar_app_info:deps(AppInfo1, Deps), pkg), - rebar_app_info:original_vsn(AppInfo2, PkgVsn1); + + %% TODO: Remove? + AppInfo2 = rebar_app_info:resource_type(AppInfo1, pkg), + + AppInfo3 = rebar_app_info:update_opts_deps(AppInfo2, Deps), + rebar_app_info:original_vsn(AppInfo3, PkgVsn1); not_found -> throw(?PRV_ERROR({missing_package, PkgName, PkgVsn})); {error, {invalid_vsn, InvalidVsn}} -> diff --git a/src/rebar_fetch.erl b/src/rebar_fetch.erl index c5c43033..fdd173fe 100644 --- a/src/rebar_fetch.erl +++ b/src/rebar_fetch.erl @@ -7,28 +7,35 @@ %% ------------------------------------------------------------------- -module(rebar_fetch). --export([lock_source/3, - download_source/3, - needs_update/3]). +-export([lock_source/2, + download_source/2, + needs_update/2]). -export([format_error/1]). -include("rebar.hrl"). -include_lib("providers/include/providers.hrl"). --spec lock_source(file:filename_all(), rebar_resource:source(), rebar_state:t()) - -> rebar_resource:source() | {error, string()}. -lock_source(AppDir, Source, State) -> - Resources = rebar_state:resources(State), - Module = get_resource_type(Source, Resources), - Module:lock(AppDir, Source). +-spec lock_source(rebar_app_info:t(), rebar_state:t()) + -> rebar_resource_v2:source() | {error, string()}. +lock_source(AppInfo, State) -> + rebar_resource_v2:lock(AppInfo, State). --spec download_source(file:filename_all(), rebar_resource:source(), rebar_state:t()) - -> true | {error, any()}. -download_source(AppDir, Source, State) -> - try download_source_(AppDir, Source, State) of +-spec download_source(rebar_app_info:t(), rebar_state:t()) + -> rebar_app_info:t() | {error, any()}. +download_source(AppInfo, State) -> + AppDir = rebar_app_info:dir(AppInfo), + try download_source_(AppInfo, State) of true -> - true; + %% freshly downloaded, update the app info opts to reflect the new config + Config = rebar_config:consult(AppDir), + AppInfo1 = rebar_app_info:update_opts(AppInfo, rebar_app_info:opts(AppInfo), Config), + case rebar_app_discover:find_app(AppInfo1, AppDir, all) of + {true, AppInfo2} -> + rebar_app_info:is_available(AppInfo2, true); + false -> + throw(?PRV_ERROR({dep_app_not_found, AppDir, rebar_app_info:name(AppInfo1)})) + end; Error -> throw(?PRV_ERROR(Error)) catch @@ -36,15 +43,14 @@ download_source(AppDir, Source, State) -> throw(?PRV_ERROR({no_resource, Location, Type})); ?WITH_STACKTRACE(C,T,S) ?DEBUG("rebar_fetch exception ~p ~p ~p", [C, T, S]), - throw(?PRV_ERROR({fetch_fail, Source})) + throw(?PRV_ERROR({fetch_fail, rebar_app_info:source(AppInfo)})) end. -download_source_(AppDir, Source, State) -> - Resources = rebar_state:resources(State), - Module = get_resource_type(Source, Resources), +download_source_(AppInfo, State) -> + AppDir = rebar_app_info:dir(AppInfo), TmpDir = ec_file:insecure_mkdtemp(), AppDir1 = rebar_utils:to_list(AppDir), - case Module:download(TmpDir, Source, State) of + case rebar_resource_v2:download(TmpDir, AppInfo, State) of {ok, _} -> ec_file:mkdir_p(AppDir1), code:del_path(filename:absname(filename:join(AppDir1, "ebin"))), @@ -56,13 +62,11 @@ download_source_(AppDir, Source, State) -> Error end. --spec needs_update(file:filename_all(), rebar_resource:source(), rebar_state:t()) +-spec needs_update(rebar_app_info:t(), rebar_state:t()) -> boolean() | {error, string()}. -needs_update(AppDir, Source, State) -> - Resources = rebar_state:resources(State), - Module = get_resource_type(Source, Resources), +needs_update(AppInfo, State) -> try - Module:needs_update(AppDir, Source) + rebar_resource_v2:needs_update(AppInfo, State) catch _:_ -> true @@ -87,25 +91,4 @@ format_error({fetch_fail, Source}) -> format_error({bad_checksum, File}) -> io_lib:format("Checksum mismatch against tarball in ~ts", [File]); format_error({bad_registry_checksum, File}) -> - io_lib:format("Checksum mismatch against registry in ~ts", [File]); -format_error({no_resource, Location, Type}) -> - io_lib:format("Cannot handle dependency ~ts.~n" - " No module for resource type ~p", [Location, Type]). - -get_resource_type({Type, Location}, Resources) -> - get_resource_module(Type, Location, Resources); -get_resource_type({Type, Location, _}, Resources) -> - get_resource_module(Type, Location, Resources); -get_resource_type({Type, _, _, Location}, Resources) -> - get_resource_module(Type, Location, Resources); -get_resource_type(_, _) -> - rebar_pkg_resource. - -get_resource_module(Type, Location, Resources) -> - case rebar_resource:find_resource_module(Type, Resources) of - {error, not_found} -> - throw({no_resource, Location, Type}); - {ok, Module} -> - Module - end. - + io_lib:format("Checksum mismatch against registry in ~ts", [File]). diff --git a/src/rebar_git_resource.erl b/src/rebar_git_resource.erl index c695ea57..b378ee8e 100644 --- a/src/rebar_git_resource.erl +++ b/src/rebar_git_resource.erl @@ -2,26 +2,30 @@ %% ex: ts=4 sw=4 et -module(rebar_git_resource). --behaviour(rebar_resource). +-behaviour(rebar_resource_v2). --export([init/1 - ,lock/2 - ,download/3 - ,needs_update/2 - ,make_vsn/1]). +-export([init/2, + lock/2, + download/4, + needs_update/2, + make_vsn/2]). -include("rebar.hrl"). %% Regex used for parsing scp style remote url -define(SCP_PATTERN, "\\A(?[^@]+)@(?[^:]+):(?.+)\\z"). --spec init(rebar_state:t()) -> {ok, term()}. -init(_State) -> - {ok, #{}}. +-spec init(atom(), rebar_state:t()) -> {ok, rebar_resource_v2:resource()}. +init(Type, _State) -> + Resource = rebar_resource_v2:new(Type, ?MODULE, #{}), + {ok, Resource}. -lock(AppDir, {git, Url, _}) -> - lock(AppDir, {git, Url}); -lock(AppDir, {git, Url}) -> +lock(AppInfo, _) -> + lock_(rebar_app_info:dir(AppInfo), rebar_app_info:source(AppInfo)). + +lock_(AppDir, {git, Url, _}) -> + lock_(AppDir, {git, Url}); +lock_(AppDir, {git, Url}) -> AbortMsg = lists:flatten(io_lib:format("Locking of git dependency failed in ~ts", [AppDir])), Dir = rebar_utils:escape_double_quotes(AppDir), {ok, VsnString} = @@ -38,14 +42,17 @@ lock(AppDir, {git, Url}) -> %% Return true if either the git url or tag/branch/ref is not the same as the currently %% checked out git repo for the dep -needs_update(Dir, {git, Url, {tag, Tag}}) -> +needs_update(AppInfo, _) -> + needs_update_(rebar_app_info:dir(AppInfo), rebar_app_info:source(AppInfo)). + +needs_update_(Dir, {git, Url, {tag, Tag}}) -> {ok, Current} = rebar_utils:sh(?FMT("git describe --tags --exact-match", []), [{cd, Dir}]), Current1 = rebar_string:trim(rebar_string:trim(Current, both, "\n"), both, "\r"), ?DEBUG("Comparing git tag ~ts with ~ts", [Tag, Current1]), not ((Current1 =:= Tag) andalso compare_url(Dir, Url)); -needs_update(Dir, {git, Url, {branch, Branch}}) -> +needs_update_(Dir, {git, Url, {branch, Branch}}) -> %% Fetch remote so we can check if the branch has changed SafeBranch = rebar_utils:escape_chars(Branch), {ok, _} = rebar_utils:sh(?FMT("git fetch origin ~ts", [SafeBranch]), @@ -55,9 +62,9 @@ needs_update(Dir, {git, Url, {branch, Branch}}) -> [{cd, Dir}]), ?DEBUG("Checking git branch ~ts for updates", [Branch]), not ((Current =:= []) andalso compare_url(Dir, Url)); -needs_update(Dir, {git, Url, "master"}) -> - needs_update(Dir, {git, Url, {branch, "master"}}); -needs_update(Dir, {git, _, Ref}) -> +needs_update_(Dir, {git, Url, "master"}) -> + needs_update_(Dir, {git, Url, {branch, "master"}}); +needs_update_(Dir, {git, _, Ref}) -> {ok, Current} = rebar_utils:sh(?FMT("git rev-parse --short=7 -q HEAD", []), [{cd, Dir}]), Current1 = rebar_string:trim(rebar_string:trim(Current, both, "\n"), @@ -103,25 +110,28 @@ parse_git_url(not_scp, Url) -> {error, Reason} end. -download(Dir, {git, Url}, State) -> +download(TmpDir, AppInfo, State, _) -> + download_(TmpDir, rebar_app_info:source(AppInfo), State). + +download_(Dir, {git, Url}, State) -> ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []), - download(Dir, {git, Url, {branch, "master"}}, State); -download(Dir, {git, Url, ""}, State) -> + download_(Dir, {git, Url, {branch, "master"}}, State); +download_(Dir, {git, Url, ""}, State) -> ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []), - download(Dir, {git, Url, {branch, "master"}}, State); -download(Dir, {git, Url, {branch, Branch}}, _State) -> + download_(Dir, {git, Url, {branch, "master"}}, State); +download_(Dir, {git, Url, {branch, Branch}}, _State) -> ok = filelib:ensure_dir(Dir), maybe_warn_local_url(Url), git_clone(branch, git_vsn(), Url, Dir, Branch); -download(Dir, {git, Url, {tag, Tag}}, _State) -> +download_(Dir, {git, Url, {tag, Tag}}, _State) -> ok = filelib:ensure_dir(Dir), maybe_warn_local_url(Url), git_clone(tag, git_vsn(), Url, Dir, Tag); -download(Dir, {git, Url, {ref, Ref}}, _State) -> +download_(Dir, {git, Url, {ref, Ref}}, _State) -> ok = filelib:ensure_dir(Dir), maybe_warn_local_url(Url), git_clone(ref, git_vsn(), Url, Dir, Ref); -download(Dir, {git, Url, Rev}, _State) -> +download_(Dir, {git, Url, Rev}, _State) -> ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []), ok = filelib:ensure_dir(Dir), maybe_warn_local_url(Url), @@ -200,7 +210,10 @@ git_vsn_fetch() -> undefined end. -make_vsn(Dir) -> +make_vsn(AppInfo, _) -> + make_vsn_(rebar_app_info:dir(AppInfo)). + +make_vsn_(Dir) -> case collect_default_refcount(Dir) of Vsn={plain, _} -> Vsn; diff --git a/src/rebar_hex_repos.erl b/src/rebar_hex_repos.erl index cba80d67..ebee1915 100644 --- a/src/rebar_hex_repos.erl +++ b/src/rebar_hex_repos.erl @@ -14,6 +14,8 @@ -include("rebar.hrl"). -include_lib("providers/include/providers.hrl"). +-export_type([repo/0]). + -type repo() :: #{name => unicode:unicode_binary(), api_url => binary(), api_key => binary(), @@ -31,8 +33,8 @@ from_state(BaseConfig, State) -> %% merge organizations parent repo options into each oraganization repo update_organizations(Repos1). --spec get_repo_config(unicode:unicode_binary(), rebar_state:t() | [hex_core:config()]) - -> {ok, hex_core:config()} | error. +-spec get_repo_config(unicode:unicode_binary(), rebar_state:t() | [repo()]) + -> {ok, repo()} | error. get_repo_config(RepoName, Repos) when is_list(Repos) -> case ec_lists:find(fun(#{name := N}) -> N =:= RepoName end, Repos) of error -> @@ -42,7 +44,7 @@ get_repo_config(RepoName, Repos) when is_list(Repos) -> end; get_repo_config(RepoName, State) -> Resources = rebar_state:resources(State), - #{repos := Repos} = rebar_resource:find_resource_state(pkg, Resources), + #{repos := Repos} = rebar_resource_v2:find_resource_state(pkg, Resources), get_repo_config(RepoName, Repos). merge_with_base_and_auth(Repos, BaseConfig, Auth) -> diff --git a/src/rebar_hg_resource.erl b/src/rebar_hg_resource.erl index 21d4a809..c61e7d3f 100644 --- a/src/rebar_hg_resource.erl +++ b/src/rebar_hg_resource.erl @@ -2,44 +2,51 @@ %% ex: ts=4 sw=4 et -module(rebar_hg_resource). --behaviour(rebar_resource). +-behaviour(rebar_resource_v2). --export([init/1 - ,lock/2 - ,download/3 - ,needs_update/2 - ,make_vsn/1]). +-export([init/2, + lock/2, + download/4, + needs_update/2, + make_vsn/2]). -include("rebar.hrl"). --spec init(rebar_state:t()) -> {ok, term()}. -init(_State) -> - {ok, #{}}. +-spec init(atom(), rebar_state:t()) -> {ok, rebar_resource_v2:resource()}. +init(Type, _State) -> + Resource = rebar_resource_v2:new(Type, ?MODULE, #{}), + {ok, Resource}. -lock(AppDir, {hg, Url, _}) -> - lock(AppDir, {hg, Url}); -lock(AppDir, {hg, Url}) -> +lock(AppInfo, _) -> + lock_(rebar_app_info:dir(AppInfo), rebar_app_info:source(AppInfo)). + +lock_(AppDir, {hg, Url, _}) -> + lock_(AppDir, {hg, Url}); +lock_(AppDir, {hg, Url}) -> Ref = get_ref(AppDir), {hg, Url, {ref, Ref}}. %% Return `true' if either the hg url or tag/branch/ref is not the same as %% the currently checked out repo for the dep -needs_update(Dir, {hg, Url, {tag, Tag}}) -> +needs_update(AppInfo, _) -> + needs_update_(rebar_app_info:dir(AppInfo), rebar_app_info:source(AppInfo)). + +needs_update_(Dir, {hg, Url, {tag, Tag}}) -> Ref = get_ref(Dir), {ClosestTag, Distance} = get_tag_distance(Dir, Ref), ?DEBUG("Comparing hg tag ~ts with ref ~ts (closest tag is ~ts at distance ~ts)", [Tag, Ref, ClosestTag, Distance]), not ((Distance =:= "0") andalso (Tag =:= ClosestTag) andalso compare_url(Dir, Url)); -needs_update(Dir, {hg, Url, {branch, Branch}}) -> +needs_update_(Dir, {hg, Url, {branch, Branch}}) -> Ref = get_ref(Dir), BRef = get_branch_ref(Dir, Branch), not ((Ref =:= BRef) andalso compare_url(Dir, Url)); -needs_update(Dir, {hg, Url, "default"}) -> +needs_update_(Dir, {hg, Url, "default"}) -> Ref = get_ref(Dir), BRef = get_branch_ref(Dir, "default"), not ((Ref =:= BRef) andalso compare_url(Dir, Url)); -needs_update(Dir, {hg, Url, Ref}) -> +needs_update_(Dir, {hg, Url, Ref}) -> LocalRef = get_ref(Dir), TargetRef = case Ref of {ref, Ref1} -> @@ -53,13 +60,16 @@ needs_update(Dir, {hg, Url, Ref}) -> ?DEBUG("Comparing hg ref ~ts with ~ts", [Ref1, LocalRef]), not ((LocalRef =:= TargetRef) andalso compare_url(Dir, Url)). -download(Dir, {hg, Url}, State) -> +download(TmpDir, AppInfo, State, _) -> + download_(TmpDir, rebar_app_info:source(AppInfo), State). + +download_(Dir, {hg, Url}, State) -> ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []), - download(Dir, {hg, Url, {branch, "default"}}, State); -download(Dir, {hg, Url, ""}, State) -> + download_(Dir, {hg, Url, {branch, "default"}}, State); +download_(Dir, {hg, Url, ""}, State) -> ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []), - download(Dir, {hg, Url, {branch, "default"}}, State); -download(Dir, {hg, Url, {branch, Branch}}, _State) -> + download_(Dir, {hg, Url, {branch, "default"}}, State); +download_(Dir, {hg, Url, {branch, Branch}}, _State) -> ok = filelib:ensure_dir(Dir), maybe_warn_local_url(Url), rebar_utils:sh(?FMT("hg clone -q -b ~ts ~ts ~ts", @@ -67,7 +77,7 @@ download(Dir, {hg, Url, {branch, Branch}}, _State) -> rebar_utils:escape_chars(Url), rebar_utils:escape_chars(filename:basename(Dir))]), [{cd, filename:dirname(Dir)}]); -download(Dir, {hg, Url, {tag, Tag}}, _State) -> +download_(Dir, {hg, Url, {tag, Tag}}, _State) -> ok = filelib:ensure_dir(Dir), maybe_warn_local_url(Url), rebar_utils:sh(?FMT("hg clone -q -u ~ts ~ts ~ts", @@ -75,7 +85,7 @@ download(Dir, {hg, Url, {tag, Tag}}, _State) -> rebar_utils:escape_chars(Url), rebar_utils:escape_chars(filename:basename(Dir))]), [{cd, filename:dirname(Dir)}]); -download(Dir, {hg, Url, {ref, Ref}}, _State) -> +download_(Dir, {hg, Url, {ref, Ref}}, _State) -> ok = filelib:ensure_dir(Dir), maybe_warn_local_url(Url), rebar_utils:sh(?FMT("hg clone -q -r ~ts ~ts ~ts", @@ -83,7 +93,7 @@ download(Dir, {hg, Url, {ref, Ref}}, _State) -> rebar_utils:escape_chars(Url), rebar_utils:escape_chars(filename:basename(Dir))]), [{cd, filename:dirname(Dir)}]); -download(Dir, {hg, Url, Rev}, _State) -> +download_(Dir, {hg, Url, Rev}, _State) -> ok = filelib:ensure_dir(Dir), maybe_warn_local_url(Url), rebar_utils:sh(?FMT("hg clone -q -r ~ts ~ts ~ts", @@ -92,7 +102,10 @@ download(Dir, {hg, Url, Rev}, _State) -> rebar_utils:escape_chars(filename:basename(Dir))]), [{cd, filename:dirname(Dir)}]). -make_vsn(Dir) -> +make_vsn(AppInfo, _) -> + make_vsn_(rebar_app_info:dir(AppInfo)). + +make_vsn_(Dir) -> BaseHg = "hg -R \"" ++ rebar_utils:escape_double_quotes(Dir) ++ "\" ", Ref = get_ref(Dir), Cmd = BaseHg ++ "log --template \"{latesttag}+build.{latesttagdistance}.rev.{node|short}\"" diff --git a/src/rebar_otp_app.erl b/src/rebar_otp_app.erl index f5bb9cf3..1754640d 100644 --- a/src/rebar_otp_app.erl +++ b/src/rebar_otp_app.erl @@ -110,7 +110,7 @@ preprocess(State, AppInfo, AppSrcFile) -> A1 = apply_app_vars(AppVars, AppData), %% AppSrcFile may contain instructions for generating a vsn number - Vsn = app_vsn(AppData, AppSrcFile, State), + Vsn = app_vsn(AppInfo, AppData, AppSrcFile, State), A2 = lists:keystore(vsn, 1, A1, {vsn, Vsn}), %% systools:make_relup/4 fails with {missing_param, registered} @@ -226,10 +226,8 @@ consult_app_file(Filename) -> end end. -app_vsn(AppData, AppFile, State) -> - AppDir = filename:dirname(filename:dirname(AppFile)), - Resources = rebar_state:resources(State), - rebar_utils:vcs_vsn(get_value(vsn, AppData, AppFile), AppDir, Resources). +app_vsn(AppInfo, AppData, AppFile, State) -> + rebar_utils:vcs_vsn(AppInfo, get_value(vsn, AppData, AppFile), State). get_value(Key, AppInfo, AppFile) -> case proplists:get_value(Key, AppInfo) of diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl index fead65a3..71a4f998 100644 --- a/src/rebar_packages.erl +++ b/src/rebar_packages.erl @@ -2,22 +2,16 @@ -export([get/2 ,get_all_names/1 - ,get_package_versions/4 - ,get_package_deps/4 - ,new_package_table/0 - ,load_and_verify_version/1 ,registry_dir/1 ,package_dir/2 - ,registry_checksum/4 ,find_highest_matching/5 - ,find_highest_matching_/5 ,verify_table/1 ,format_error/1 ,update_package/3 ,resolve_version/5]). -ifdef(TEST). --export([cmp_/4, cmpl_/4, valid_vsn/1]). +-export([new_package_table/0, find_highest_matching_/5, cmp_/4, cmpl_/4, valid_vsn/1]). -endif. -export_type([package/0]). @@ -35,7 +29,7 @@ format_error({missing_package, Name, Vsn}) -> format_error({missing_package, Pkg}) -> io_lib:format("Package not found in any repo: ~p.", [Pkg]). --spec get(hex_core:config(), binary()) -> {ok, map()} | {error, term()}. +-spec get(rebar_hex_repos:repo(), binary()) -> {ok, map()} | {error, term()}. get(Config, Name) -> try hex_api_package:get(Config, Name) of {ok, {200, _Headers, PkgInfo}} -> @@ -79,7 +73,7 @@ get_package(Dep, Vsn, Hash, Repo, Table, State) -> -> {ok, #package{}} | not_found. get_package(Dep, Vsn, undefined, Retired, Repo, Table, State) -> get_package(Dep, Vsn, '_', Retired, Repo, Table, State); -get_package(Dep, Vsn, Hash, Retired, Repos, Table, State) when is_list(Repos) -> +get_package(Dep, Vsn, Hash, Retired, Repos, Table, State) -> ?MODULE:verify_table(State), case ets:select(Table, [{#package{key={Dep, Vsn, Repo}, checksum=Hash, @@ -90,35 +84,12 @@ get_package(Dep, Vsn, Hash, Retired, Repos, Table, State) when is_list(Repos) -> {ok, Package}; _ -> not_found - end; -get_package(Dep, Vsn, Hash, Retired, Repo, Table, State) -> - get_package(Dep, Vsn, Hash, Retired, [Repo], Table, State). + end. new_package_table() -> ?PACKAGE_TABLE = ets:new(?PACKAGE_TABLE, [named_table, public, ordered_set, {keypos, 2}]), ets:insert(?PACKAGE_TABLE, {?PACKAGE_INDEX_VERSION, package_index_version}). --spec get_package_deps(unicode:unicode_binary(), unicode:unicode_binary(), vsn(), rebar_state:t()) - -> [map()]. -get_package_deps(Name, Vsn, Repo, State) -> - try_lookup(?PACKAGE_TABLE, {Name, Vsn, Repo}, #package.dependencies, State). - --spec registry_checksum(unicode:unicode_binary(), vsn(), unicode:unicode_binary(), rebar_state:t()) - -> binary(). -registry_checksum(Name, Vsn, Repo, State) -> - try_lookup(?PACKAGE_TABLE, {Name, Vsn, Repo}, #package.checksum, State). - -try_lookup(Table, Key={_, _, Repo}, Element, State) -> - ?MODULE:verify_table(State), - try - ets:lookup_element(Table, Key, Element) - catch - _:_ -> - handle_missing_package(Key, Repo, State, fun(_) -> - ets:lookup_element(Table, Key, Element) - end) - end. - load_and_verify_version(State) -> {ok, RegistryDir} = registry_dir(State), case ets:file2tab(filename:join(RegistryDir, ?INDEX_FILE)) of @@ -171,16 +142,14 @@ registry_dir(State) -> end, {ok, RegistryDir}. +-spec package_dir(rebar_hex_repos:repo(), rebar_state:t()) -> {ok, filename:filename_all()}. package_dir(Repo, State) -> - case registry_dir(State) of - {ok, RegistryDir} -> - RepoName = maps:get(name, Repo), - PackageDir = filename:join([RegistryDir, rebar_utils:to_list(RepoName), "packages"]), - ok = filelib:ensure_dir(filename:join(PackageDir, "placeholder")), - {ok, PackageDir}; - Error -> - Error - end. + {ok, RegistryDir} = registry_dir(State), + RepoName = maps:get(name, Repo), + PackageDir = filename:join([RegistryDir, rebar_utils:to_list(RepoName), "packages"]), + ok = filelib:ensure_dir(filename:join(PackageDir, "placeholder")), + {ok, PackageDir}. + %% Hex supports use of ~> to specify the version required for a dependency. %% Since rebar3 requires exact versions to choose from we find the highest @@ -306,7 +275,7 @@ insert_releases(Name, Releases, Repo, Table) -> %% if checksum is defined search for any matching repo matching pkg-vsn and checksum resolve_version(Dep, DepVsn, Hash, HexRegistry, State) when is_binary(Hash) -> Resources = rebar_state:resources(State), - #{repos := RepoConfigs} = rebar_resource:find_resource_state(pkg, Resources), + #{repos := RepoConfigs} = rebar_resource_v2:find_resource_state(pkg, Resources), RepoNames = [RepoName || #{name := RepoName} <- RepoConfigs], %% allow retired packages when we have a checksum @@ -358,7 +327,7 @@ check_all_repos(Fun, RepoConfigs) -> handle_missing_no_exception(Fun, Dep, State) -> Resources = rebar_state:resources(State), - #{repos := RepoConfigs} = rebar_resource:find_resource_state(pkg, Resources), + #{repos := RepoConfigs} = rebar_resource_v2:find_resource_state(pkg, Resources), %% first check all repos in order for a local match %% if none is found then we step through checking after updating the repo registry diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl index 07039bfe..5d7f4856 100644 --- a/src/rebar_pkg_resource.erl +++ b/src/rebar_pkg_resource.erl @@ -2,21 +2,18 @@ %% ex: ts=4 sw=4 et -module(rebar_pkg_resource). --behaviour(rebar_resource). +-behaviour(rebar_resource_v2). --export([init/1 - ,lock/2 - ,download/3 - ,download/4 - ,needs_update/2 - ,make_vsn/1]). - --export([request/4 - ,etag/1]). +-export([init/2, + lock/2, + download/4, + download/5, + needs_update/2, + make_vsn/2]). -ifdef(TEST). %% exported for test purposes --export([store_etag_in_cache/2]). +-export([store_etag_in_cache/2, etag/1, request/4]). -endif. -include("rebar.hrl"). @@ -34,22 +31,26 @@ %% Public API %%============================================================================== --spec init(rebar_state:t()) -> {ok, term()}. -init(State) -> +-spec init(atom(), rebar_state:t()) -> {ok, rebar_resource_v2:resource()}. +init(Type, State) -> {ok, Vsn} = application:get_key(rebar, vsn), BaseConfig = #{http_adapter => hex_http_httpc, http_user_agent_fragment => <<"(rebar3/", (list_to_binary(Vsn))/binary, ") (httpc)">>, http_adapter_config => #{profile => rebar}}, Repos = rebar_hex_repos:from_state(BaseConfig, State), - {ok, #{repos => Repos, - base_config => BaseConfig}}. + Resource = rebar_resource_v2:new(Type, ?MODULE, #{repos => Repos, + base_config => BaseConfig}), + {ok, Resource}. + + --spec lock(AppDir, Source) -> Res when - AppDir :: file:name(), - Source :: tuple(), +-spec lock(AppInfo, ResourceState) -> Res when + AppInfo :: rebar_app_info:t(), + ResourceState :: rebar_resource_v2:resource_state(), Res :: {atom(), string(), any(), binary()}. -lock(_AppDir, {pkg, Name, Vsn, Hash, _RepoConfig}) -> +lock(AppInfo, _) -> + {pkg, Name, Vsn, Hash, _RepoConfig} = rebar_app_info:source(AppInfo), {pkg, Name, Vsn, Hash}. %%------------------------------------------------------------------------------ @@ -58,12 +59,12 @@ lock(_AppDir, {pkg, Name, Vsn, Hash, _RepoConfig}) -> %% version. %% @end %%------------------------------------------------------------------------------ --spec needs_update(Dir, Pkg) -> Res when - Dir :: file:name(), - Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary(), RepoConfig :: hex_core:config()}, +-spec needs_update(AppInfo, ResourceState) -> Res when + AppInfo :: rebar_app_info:t(), + ResourceState :: rebar_resource_v2:resource_state(), Res :: boolean(). -needs_update(Dir, {pkg, _Name, Vsn, _Hash, _}) -> - [AppInfo] = rebar_app_discover:find_apps([Dir], all), +needs_update(AppInfo, _) -> + {pkg, _Name, Vsn, _Hash, _} = rebar_app_info:source(AppInfo), case rebar_app_info:original_vsn(AppInfo) =:= rebar_utils:to_binary(Vsn) of true -> false; @@ -76,13 +77,14 @@ needs_update(Dir, {pkg, _Name, Vsn, _Hash, _}) -> %% Download the given pkg. %% @end %%------------------------------------------------------------------------------ --spec download(TmpDir, Pkg, State) -> Res when +-spec download(TmpDir, AppInfo, State, ResourceState) -> Res when TmpDir :: file:name(), - Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary(), RepoConfig :: hex_core:config()}, + AppInfo :: rebar_app_info:t(), + ResourceState :: rebar_resource_v2:resource_state(), State :: rebar_state:t(), Res :: {'error',_} | {'ok',_} | {'tarball',binary() | string()}. -download(TmpDir, Pkg, State) -> - download(TmpDir, Pkg, State, true). +download(TmpDir, AppInfo, State, ResourceState) -> + download(TmpDir, rebar_app_info:source(AppInfo), State, ResourceState, true). %%------------------------------------------------------------------------------ %% @doc @@ -91,13 +93,14 @@ download(TmpDir, Pkg, State) -> %% is different. %% @end %%------------------------------------------------------------------------------ --spec download(TmpDir, Pkg, State, UpdateETag) -> Res when +-spec download(TmpDir, Pkg, State, ResourceState, UpdateETag) -> Res when TmpDir :: file:name(), - Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary(), RepoConfig :: hex_core:config()}, + Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary(), RepoConfig :: rebar_hex_repos:repo()}, State :: rebar_state:t(), + ResourceState:: rebar_resource_v2:resource_state(), UpdateETag :: boolean(), Res :: download_result(). -download(TmpDir, Pkg={pkg, Name, Vsn, _Hash, Repo}, State, UpdateETag) -> +download(TmpDir, Pkg={pkg, Name, Vsn, _Hash, Repo}, State, _ResourceState, UpdateETag) -> {ok, PackageDir} = rebar_packages:package_dir(Repo, State), Package = binary_to_list(<>), ETagFile = binary_to_list(<>), @@ -112,10 +115,11 @@ download(TmpDir, Pkg={pkg, Name, Vsn, _Hash, Repo}, State, UpdateETag) -> %% Returns {error, string()} as this operation is not supported for pkg sources. %% @end %%------------------------------------------------------------------------------ --spec make_vsn(Vsn) -> Res when - Vsn :: any(), - Res :: {'error',[1..255,...]}. -make_vsn(_) -> +-spec make_vsn(AppInfo, ResourceState) -> Res when + AppInfo :: rebar_app_info:t(), + ResourceState :: rebar_resource_v2:resource_state(), + Res :: {'error', string()}. +make_vsn(_, _) -> {error, "Replacing version of type pkg not supported."}. %%------------------------------------------------------------------------------ @@ -126,7 +130,7 @@ make_vsn(_) -> %% {ok, Contents, NewEtag}, otherwise if some error occured return error. %% @end %%------------------------------------------------------------------------------ --spec request(hex_core:config(), binary(), binary(), false | binary()) +-spec request(rebar_hex_repos:repo(), binary(), binary(), false | binary()) -> {ok, cached} | {ok, binary(), binary()} | error. request(Config, Name, Version, ETag) -> Config1 = Config#{http_etag => ETag}, @@ -184,7 +188,7 @@ store_etag_in_cache(Path, ETag) -> UpdateETag) -> Res when TmpDir :: file:name(), CachePath :: file:name(), - Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary(), RepoConfig :: hex_core:config()}, + Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary(), RepoConfig :: rebar_hex_repos:repo()}, ETag :: binary(), State :: rebar_state:t(), ETagPath :: file:name(), @@ -212,7 +216,7 @@ cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn, _Hash, RepoConfig}, ETag -spec serve_from_cache(TmpDir, CachePath, Pkg, State) -> Res when TmpDir :: file:name(), CachePath :: file:name(), - Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary(), RepoConfig :: hex_core:config()}, + Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary(), RepoConfig :: rebar_hex_repos:repo()}, State :: rebar_state:t(), Res :: cached_result(). serve_from_cache(TmpDir, CachePath, Pkg, State) -> @@ -237,7 +241,7 @@ serve_from_cache(TmpDir, CachePath, Pkg, State) -> ETagPath) -> Res when TmpDir :: file:name(), CachePath :: file:name(), - Package :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary(), RepoConfig :: hex_core:config()}, + Package :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary(), RepoConfig :: rebar_hex_repos:repo()}, ETag :: binary(), Binary :: binary(), State :: rebar_state:t(), @@ -272,7 +276,7 @@ extract(TmpDir, CachePath) -> {Files, Contents, Version, Meta}. -spec checksums(Pkg, Files, Contents, Version, Meta, State) -> Res when - Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary(), RepoConfig :: hex_core:config()}, + Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary(), RepoConfig :: rebar_hex_repos:repo()}, Files :: list({file:name(), binary()}), Contents :: binary(), Version :: binary(), diff --git a/src/rebar_prv_deps.erl b/src/rebar_prv_deps.erl index a88b0147..577a859d 100644 --- a/src/rebar_prv_deps.erl +++ b/src/rebar_prv_deps.erl @@ -97,10 +97,11 @@ display_dep(_State, {Name, _Vsn, Source}) when is_tuple(Source) -> display_dep(_State, {Name, _Vsn, Source, _Opts}) when is_tuple(Source) -> ?CONSOLE("~ts* (~ts source)", [rebar_utils:to_binary(Name), type(Source)]); %% Locked -display_dep(State, {Name, Source={pkg, _, Vsn}, Level}) when is_integer(Level) -> +display_dep(State, {Name, _Source={pkg, _, Vsn}, Level}) when is_integer(Level) -> DepsDir = rebar_dir:deps_dir(State), AppDir = filename:join([DepsDir, rebar_utils:to_binary(Name)]), - NeedsUpdate = case rebar_fetch:needs_update(AppDir, Source, State) of + {ok, AppInfo} = rebar_app_info:discover(AppDir), + NeedsUpdate = case rebar_fetch:needs_update(AppInfo, State) of true -> "*"; false -> "" end, @@ -108,7 +109,8 @@ display_dep(State, {Name, Source={pkg, _, Vsn}, Level}) when is_integer(Level) - display_dep(State, {Name, Source, Level}) when is_tuple(Source), is_integer(Level) -> DepsDir = rebar_dir:deps_dir(State), AppDir = filename:join([DepsDir, rebar_utils:to_binary(Name)]), - NeedsUpdate = case rebar_fetch:needs_update(AppDir, Source, State) of + {ok, AppInfo} = rebar_app_info:discover(AppDir), + NeedsUpdate = case rebar_fetch:needs_update(AppInfo, State) of true -> "*"; false -> "" end, diff --git a/src/rebar_prv_deps_tree.erl b/src/rebar_prv_deps_tree.erl index 07c79724..d7b49c53 100644 --- a/src/rebar_prv_deps_tree.erl +++ b/src/rebar_prv_deps_tree.erl @@ -39,18 +39,16 @@ format_error(Reason) -> %% Internal functions print_deps_tree(SrcDeps, Verbose, State) -> - Resources = rebar_state:resources(State), D = lists:foldl(fun(App, Dict) -> Name = rebar_app_info:name(App), Vsn = rebar_app_info:original_vsn(App), - AppDir = rebar_app_info:dir(App), - Vsn1 = rebar_utils:vcs_vsn(Vsn, AppDir, Resources), + Vsn1 = rebar_utils:vcs_vsn(App, Vsn, State), Source = rebar_app_info:source(App), Parent = rebar_app_info:parent(App), dict:append_list(Parent, [{Name, Vsn1, Source}], Dict) end, dict:new(), SrcDeps), ProjectAppNames = [{rebar_app_info:name(App) - ,rebar_utils:vcs_vsn(rebar_app_info:original_vsn(App), rebar_app_info:dir(App), Resources) + ,rebar_utils:vcs_vsn(App, rebar_app_info:original_vsn(App), State) ,project} || App <- rebar_state:project_apps(State)], case dict:find(root, D) of {ok, Children} -> diff --git a/src/rebar_prv_install_deps.erl b/src/rebar_prv_install_deps.erl index 046bca28..190f0a8f 100644 --- a/src/rebar_prv_install_deps.erl +++ b/src/rebar_prv_install_deps.erl @@ -277,10 +277,8 @@ update_unseen_dep(AppInfo, Profile, Level, Deps, Apps, State, Upgrade, Seen, Loc -spec handle_dep(rebar_state:t(), atom(), file:filename_all(), rebar_app_info:t(), list(), integer()) -> {rebar_app_info:t(), [rebar_app_info:t()], rebar_state:t()}. handle_dep(State, Profile, DepsDir, AppInfo, Locks, Level) -> Name = rebar_app_info:name(AppInfo), - C = rebar_config:consult(rebar_app_info:dir(AppInfo)), - AppInfo0 = rebar_app_info:update_opts(AppInfo, rebar_app_info:opts(AppInfo), C), - AppInfo1 = rebar_app_info:apply_overrides(rebar_app_info:get(AppInfo, overrides, []), AppInfo0), + AppInfo1 = rebar_app_info:apply_overrides(rebar_app_info:get(AppInfo, overrides, []), AppInfo), AppInfo2 = rebar_app_info:apply_profiles(AppInfo1, [default, prod]), Plugins = rebar_app_info:get(AppInfo2, plugins, []), @@ -297,34 +295,33 @@ handle_dep(State, Profile, DepsDir, AppInfo, Locks, Level) -> AppInfo4 = rebar_app_info:deps(AppInfo3, rebar_state:deps_names(Deps)), %% Keep all overrides from the global config and this dep when parsing its deps - Overrides = rebar_app_info:get(AppInfo0, overrides, []), + Overrides = rebar_app_info:get(AppInfo, overrides, []), Deps1 = rebar_app_utils:parse_deps(Name, DepsDir, Deps, rebar_state:set(State, overrides, Overrides) ,Locks, Level+1), {AppInfo4, Deps1, State1}. -spec maybe_fetch(rebar_app_info:t(), atom(), boolean(), - sets:set(binary()), rebar_state:t()) -> {boolean(), rebar_app_info:t()}. + sets:set(binary()), rebar_state:t()) -> {ok, rebar_app_info:t()}. maybe_fetch(AppInfo, Profile, Upgrade, Seen, State) -> AppDir = rebar_utils:to_list(rebar_app_info:dir(AppInfo)), %% Don't fetch dep if it exists in the _checkouts dir case rebar_app_info:is_checkout(AppInfo) of true -> - {false, AppInfo}; + {ok, AppInfo}; false -> - case rebar_app_discover:find_app(AppInfo, AppDir, all) of + case rebar_app_info:is_available(AppInfo) of false -> - true = fetch_app(AppInfo, AppDir, State), - maybe_symlink_default(State, Profile, AppDir, AppInfo), - {true, rebar_app_info:valid(update_app_info(AppDir, AppInfo), false)}; - {true, AppInfo1} -> - case sets:is_element(rebar_app_info:name(AppInfo1), Seen) of + AppInfo1 = fetch_app(AppInfo, State), + maybe_symlink_default(State, Profile, AppDir, AppInfo1), + {ok, rebar_app_info:is_available(rebar_app_info:valid(AppInfo1, false), true)}; + true -> + case sets:is_element(rebar_app_info:name(AppInfo), Seen) of true -> - {false, AppInfo1}; + {ok, AppInfo}; false -> - maybe_symlink_default(State, Profile, AppDir, AppInfo1), - MaybeUpgrade = maybe_upgrade(AppInfo, AppDir, Upgrade, State), - AppInfo2 = update_app_info(AppDir, AppInfo1), - {MaybeUpgrade, AppInfo2} + maybe_symlink_default(State, Profile, AppDir, AppInfo), + AppInfo1 = maybe_upgrade(AppInfo, AppDir, Upgrade, State), + {ok, AppInfo1} end end end. @@ -372,43 +369,30 @@ make_relative_to_root(State, Path) when is_list(Path) -> Root = rebar_dir:root_dir(State), rebar_dir:make_relative_path(Path, Root). -fetch_app(AppInfo, AppDir, State) -> +fetch_app(AppInfo, State) -> ?INFO("Fetching ~ts (~p)", [rebar_app_info:name(AppInfo), - rebar_resource:format_source(rebar_app_info:source(AppInfo))]), - Source = rebar_app_info:source(AppInfo), - true = rebar_fetch:download_source(AppDir, Source, State). - -%% This is called after the dep has been downloaded and unpacked, if it hadn't been already. -%% So this is the first time for newly downloaded apps that its .app/.app.src data can -%% be read in an parsed. -update_app_info(AppDir, AppInfo) -> - case rebar_app_discover:find_app(AppInfo, AppDir, all) of - {true, AppInfo1} -> - AppInfo1; - false -> - throw(?PRV_ERROR({dep_app_not_found, AppDir, rebar_app_info:name(AppInfo)})) - end. + rebar_resource_v2:format_source(rebar_app_info:source(AppInfo))]), + rebar_fetch:download_source(AppInfo, State). -maybe_upgrade(AppInfo, AppDir, Upgrade, State) -> - Source = rebar_app_info:source(AppInfo), +maybe_upgrade(AppInfo, _AppDir, Upgrade, State) -> case Upgrade orelse rebar_app_info:is_lock(AppInfo) of true -> - case rebar_fetch:needs_update(AppDir, Source, State) of + case rebar_fetch:needs_update(AppInfo, State) of true -> ?INFO("Upgrading ~ts (~p)", [rebar_app_info:name(AppInfo), - rebar_resource:format_source(rebar_app_info:source(AppInfo))]), - true = rebar_fetch:download_source(AppDir, Source, State); + rebar_resource_v2:format_source(rebar_app_info:source(AppInfo))]), + rebar_fetch:download_source(AppInfo, State); false -> case Upgrade of true -> ?INFO("No upgrade needed for ~ts", [rebar_app_info:name(AppInfo)]), - false; + AppInfo; false -> - false + AppInfo end end; false -> - false + AppInfo end. warn_skip_deps(AppInfo, State) -> diff --git a/src/rebar_prv_lock.erl b/src/rebar_prv_lock.erl index cbe8dfef..570c03f4 100644 --- a/src/rebar_prv_lock.erl +++ b/src/rebar_prv_lock.erl @@ -54,12 +54,9 @@ format_error(Reason) -> build_locks(State) -> AllDeps = rebar_state:lock(State), [begin - Dir = rebar_app_info:dir(Dep), - Source = rebar_app_info:source(Dep), - %% If source is tuple it is a source dep %% e.g. {git, "git://github.com/ninenines/cowboy.git", "master"} - {rebar_app_info:name(Dep) - ,rebar_fetch:lock_source(Dir, Source, State) - ,rebar_app_info:dep_level(Dep)} + {rebar_app_info:name(Dep), + rebar_fetch:lock_source(Dep, State), + rebar_app_info:dep_level(Dep)} end || Dep <- AllDeps, not(rebar_app_info:is_checkout(Dep))]. diff --git a/src/rebar_prv_packages.erl b/src/rebar_prv_packages.erl index 37a98a59..3e54cdc2 100644 --- a/src/rebar_prv_packages.erl +++ b/src/rebar_prv_packages.erl @@ -35,7 +35,7 @@ do(State) -> ?PRV_ERROR(no_package_arg); Name -> Resources = rebar_state:resources(State), - #{repos := Repos} = rebar_resource:find_resource_state(pkg, Resources), + #{repos := Repos} = rebar_resource_v2:find_resource_state(pkg, Resources), Results = get_package(rebar_utils:to_binary(Name), Repos), case lists:all(fun({_, {error, not_found}}) -> true; (_) -> false end, Results) of true -> @@ -46,6 +46,7 @@ do(State) -> end end. +-spec get_package(binary(), [map()]) -> [{binary(), {ok, map()} | {error, term()}}]. get_package(Name, Repos) -> lists:foldl(fun(RepoConfig, Acc) -> [{maps:get(name, RepoConfig), rebar_packages:get(RepoConfig, Name)} | Acc] diff --git a/src/rebar_prv_repos.erl b/src/rebar_prv_repos.erl index 6f4bad33..05159106 100644 --- a/src/rebar_prv_repos.erl +++ b/src/rebar_prv_repos.erl @@ -35,7 +35,7 @@ init(State) -> -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. do(State) -> Resources = rebar_state:resources(State), - #{repos := Repos} = rebar_resource:find_resource_state(pkg, Resources), + #{repos := Repos} = rebar_resource_v2:find_resource_state(pkg, Resources), ?CONSOLE("Repos:", []), %%TODO: do some formatting diff --git a/src/rebar_prv_update.erl b/src/rebar_prv_update.erl index b384a6c9..4c820c5b 100644 --- a/src/rebar_prv_update.erl +++ b/src/rebar_prv_update.erl @@ -35,7 +35,7 @@ init(State) -> do(State) -> Names = rebar_packages:get_all_names(State), Resources = rebar_state:resources(State), - #{repos := RepoConfigs} = rebar_resource:find_resource_state(pkg, Resources), + #{repos := RepoConfigs} = rebar_resource_v2:find_resource_state(pkg, Resources), [[update_package(Name, RepoConfig, State) || Name <- Names] || RepoConfig <- RepoConfigs], diff --git a/src/rebar_prv_upgrade.erl b/src/rebar_prv_upgrade.erl index 82502963..b1b1b165 100644 --- a/src/rebar_prv_upgrade.erl +++ b/src/rebar_prv_upgrade.erl @@ -135,7 +135,7 @@ update_pkg_deps([{Name, _, _} | Rest], AppInfos, State) -> case element(1, rebar_app_info:source(AppInfo)) of pkg -> Resources = rebar_state:resources(State), - #{repos := RepoConfigs} = rebar_resource:find_resource_state(pkg, Resources), + #{repos := RepoConfigs} = rebar_resource_v2:find_resource_state(pkg, Resources), [update_package(Name, RepoConfig, State) || RepoConfig <- RepoConfigs]; _ -> skip diff --git a/src/rebar_resource.erl b/src/rebar_resource.erl index ed8a2170..0954df76 100644 --- a/src/rebar_resource.erl +++ b/src/rebar_resource.erl @@ -3,27 +3,23 @@ -module(rebar_resource). -export([new/3, - find_resource_module/2, - find_resource_state/2, - format_source/1]). + lock/2, + download/4, + needs_update/2, + make_vsn/2]). --export_type([resource/0 - ,source/0 - ,type/0 - ,location/0 - ,ref/0]). +-export_type([source/0, + type/0, + location/0, + ref/0]). --record(resource, {type :: atom(), - module :: module(), - state :: term()}). +-include("rebar.hrl"). --type resource() :: #resource{}. -type source() :: {type(), location(), ref()} | {type(), location(), ref(), binary()}. -type type() :: atom(). -type location() :: string(). -type ref() :: any(). --callback init(rebar_state:t()) -> {ok, term()}. -callback lock(file:filename_all(), tuple()) -> source(). -callback download(file:filename_all(), tuple(), rebar_state:t()) -> @@ -33,36 +29,21 @@ -callback make_vsn(file:filename_all()) -> {plain, string()} | {error, string()}. --optional_callbacks([init/1]). - --spec new(type(), module(), term()) -> resource(). +-spec new(type(), module(), term()) -> rebar_resource_v2:resource(). new(Type, Module, State) -> #resource{type=Type, module=Module, - state=State}. + state=State, + implementation=?MODULE}. + +lock(Module, AppInfo) -> + Module:lock(rebar_app_info:dir(AppInfo), rebar_app_info:source(AppInfo)). -find_resource_module(Type, Resources) -> - case lists:keyfind(Type, #resource.type, Resources) of - false when is_atom(Type) -> - case code:which(Type) of - non_existing -> - {error, not_found}; - _ -> - {ok, Type} - end; - false -> - {error, not_found}; - #resource{module=Module} -> - {ok, Module} - end. +download(Module, TmpDir, AppInfo, State) -> + Module:download(TmpDir, rebar_app_info:source(AppInfo), State). -find_resource_state(Type, Resources) -> - case lists:keyfind(Type, #resource.type, Resources) of - false -> - {error, not_found}; - #resource{state=State} -> - State - end. +needs_update(Module, AppInfo) -> + Module:needs_update(rebar_app_info:dir(AppInfo), rebar_app_info:source(AppInfo)). -format_source({pkg, Name, Vsn, _Hash, _}) -> {pkg, Name, Vsn}; -format_source(Source) -> Source. +make_vsn(Module, AppInfo) -> + Module:make_vsn(rebar_app_info:dir(AppInfo)). diff --git a/src/rebar_resource_v2.erl b/src/rebar_resource_v2.erl new file mode 100644 index 00000000..ef897872 --- /dev/null +++ b/src/rebar_resource_v2.erl @@ -0,0 +1,147 @@ +%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ts=4 sw=4 et +-module(rebar_resource_v2). + +-export([new/3, + find_resource_state/2, + format_source/1, + lock/2, + download/3, + needs_update/2, + make_vsn/3, + format_error/1]). + +-export_type([resource/0, + source/0, + type/0, + location/0, + ref/0, + resource_state/0]). + +-include("rebar.hrl"). +-include_lib("providers/include/providers.hrl"). + +-type resource() :: #resource{}. +-type source() :: {type(), location(), ref()} | {type(), location(), ref(), binary()}. +-type type() :: atom(). +-type location() :: string(). +-type ref() :: any(). +-type resource_state() :: term(). + +-callback init(type(), rebar_state:t()) -> {ok, resource()}. +-callback lock(rebar_app_info:t(), resource_state()) -> source(). +-callback download(file:filename_all(), rebar_app_info:t(), resource_state(), rebar_state:t()) -> + {tarball, file:filename_all()} | {ok, any()} | {error, any()}. +-callback needs_update(rebar_app_info:t(), resource_state()) -> boolean(). +-callback make_vsn(rebar_app_info:t(), resource_state()) -> + {plain, string()} | {error, string()}. + +-spec new(type(), module(), term()) -> resource(). +new(Type, Module, State) -> + #resource{type=Type, + module=Module, + state=State, + implementation=?MODULE}. + +-spec find_resource(type(), [resource()]) -> {ok, resource()} | {error, not_found}. +find_resource(Type, Resources) -> + case ec_lists:find(fun(#resource{type=T}) -> T =:= Type end, Resources) of + error when is_atom(Type) -> + case code:which(Type) of + non_existing -> + {error, not_found}; + _ -> + {ok, rebar_resource:new(Type, Type, #{})} + end; + error -> + {error, not_found}; + {ok, Resource} -> + {ok, Resource} + end. + +find_resource_state(Type, Resources) -> + case lists:keyfind(Type, #resource.type, Resources) of + false -> + {error, not_found}; + #resource{state=State} -> + State + end. + +format_source({pkg, Name, Vsn, _Hash, _}) -> {pkg, Name, Vsn}; +format_source(Source) -> Source. + +lock(AppInfo, State) -> + resource_run(lock, rebar_app_info:source(AppInfo), [AppInfo], State). + +resource_run(Function, Source, Args, State) -> + Resources = rebar_state:resources(State), + case get_resource_type(Source, Resources) of + {ok, #resource{type=_, + module=Module, + state=ResourceState, + implementation=?MODULE}} -> + erlang:apply(Module, Function, Args++[ResourceState]); + {ok, #resource{type=_, + module=Module, + state=_, + implementation=rebar_resource}} -> + erlang:apply(rebar_resource, Function, [Module | Args]) + end. + +download(TmpDir, AppInfo, State) -> + resource_run(download, rebar_app_info:source(AppInfo), [TmpDir, AppInfo, State], State). + +needs_update(AppInfo, State) -> + resource_run(needs_update, rebar_app_info:source(AppInfo), [AppInfo], State). + +%% this is a special case since it is used for project apps as well, not just deps +make_vsn(AppInfo, VcsType, State) -> + Resources = rebar_state:resources(State), + case is_resource_type(VcsType, Resources) of + true -> + case find_resource(VcsType, Resources) of + {ok, #resource{type=_, + module=Module, + state=ResourceState, + implementation=?MODULE}} -> + Module:make_vsn(AppInfo, ResourceState); + {ok, #resource{type=_, + module=Module, + state=_, + implementation=rebar_resource}} -> + rebar_resource:make_vsn(Module, AppInfo) + end; + false -> + unknown + end. + +format_error({no_resource, Location, Type}) -> + io_lib:format("Cannot handle dependency ~ts.~n" + " No module found for resource type ~p.", [Location, Type]); +format_error({no_resource, Source}) -> + io_lib:format("Cannot handle dependency ~ts.~n" + " No module found for unknown resource type.", [Source]). + +is_resource_type(Type, Resources) -> + lists:any(fun(#resource{type=T}) -> T =:= Type end, Resources). + +-spec get_resource_type(term(), [resource()]) -> {ok, resource()}. +get_resource_type({Type, Location}, Resources) -> + get_resource(Type, Location, Resources); +get_resource_type({Type, Location, _}, Resources) -> + get_resource(Type, Location, Resources); +get_resource_type({Type, _, _, Location}, Resources) -> + get_resource(Type, Location, Resources); +get_resource_type(Location={Type, _, _, _, _}, Resources) -> + get_resource(Type, Location, Resources); +get_resource_type(Source, _) -> + throw(?PRV_ERROR({no_resource, Source})). + +-spec get_resource(type(), term(), [resource()]) -> {ok, resource()}. +get_resource(Type, Location, Resources) -> + case find_resource(Type, Resources) of + {error, not_found} -> + throw(?PRV_ERROR({no_resource, Location, Type})); + {ok, Resource} -> + {ok, Resource} + end. diff --git a/src/rebar_state.erl b/src/rebar_state.erl index 9a704e86..ec74eea4 100644 --- a/src/rebar_state.erl +++ b/src/rebar_state.erl @@ -353,28 +353,44 @@ namespace(#state_t{namespace=Namespace}) -> namespace(State=#state_t{}, Namespace) -> State#state_t{namespace=Namespace}. --spec resources(t()) -> [{rebar_resource:type(), module()}]. +-spec resources(t()) -> [{rebar_resource_v2:type(), module()}]. resources(#state_t{resources=Resources}) -> Resources. --spec resources(t(), [{rebar_resource:type(), module()}]) -> t(). +-spec resources(t(), [{rebar_resource_v2:type(), module()}]) -> t(). resources(State, NewResources) -> lists:foldl(fun(Resource, StateAcc) -> add_resource(StateAcc, Resource) end, State, NewResources). --spec add_resource(t(), {rebar_resource:type(), module()}) -> t(). +-spec add_resource(t(), {rebar_resource_v2:type(), module()}) -> t(). add_resource(State=#state_t{resources=Resources}, {ResourceType, ResourceModule}) -> _ = code:ensure_loaded(ResourceModule), - {ok, ResourceState} = case erlang:function_exported(ResourceModule, init, 1) of - true -> - ResourceModule:init(State); - false -> - {ok, #{}} - end, - State#state_t{resources=[rebar_resource:new(ResourceType, - ResourceModule, - ResourceState) | Resources]}. + Resource = case erlang:function_exported(ResourceModule, init, 2) of + true -> + case ResourceModule:init(ResourceType, State) of + {ok, R=#resource{}} -> + R; + _ -> + %% init didn't return a resource + %% must be an old resource + warn_old_resource(ResourceModule), + rebar_resource:new(ResourceType, + ResourceModule, + #{}) + end; + false -> + %% no init, must be initial implementation + warn_old_resource(ResourceModule), + rebar_resource:new(ResourceType, + ResourceModule, + #{}) + end, + State#state_t{resources=[Resource | Resources]}. + +warn_old_resource(ResourceModule) -> + ?WARN("Using custom resource ~s that implements a deprecated api. " + "It should be upgraded to rebar_resource_v2.", [ResourceModule]). create_resources(Resources, State) -> lists:foldl(fun(R, StateAcc) -> diff --git a/src/rebar_utils.erl b/src/rebar_utils.erl index 174de90b..e159acfe 100644 --- a/src/rebar_utils.erl +++ b/src/rebar_utils.erl @@ -663,12 +663,12 @@ escript_foldl(Fun, Acc, File) -> Error end. -vcs_vsn(Vcs, Dir, Resources) -> - case vcs_vsn_cmd(Vcs, Dir, Resources) of +vcs_vsn(AppInfo, Vcs, State) -> + case vcs_vsn_cmd(AppInfo, Vcs, State) of {plain, VsnString} -> VsnString; {cmd, CmdString} -> - vcs_vsn_invoke(CmdString, Dir); + vcs_vsn_invoke(CmdString, rebar_app_info:dir(AppInfo)); unknown -> ?ABORT("vcs_vsn: Unknown vsn format: ~p", [Vcs]); {error, Reason} -> @@ -676,23 +676,18 @@ vcs_vsn(Vcs, Dir, Resources) -> end. %% Temp work around for repos like relx that use "semver" -vcs_vsn_cmd(Vsn, _, _) when is_binary(Vsn) -> +vcs_vsn_cmd(_AppInfo, Vsn, _) when is_binary(Vsn) -> {plain, Vsn}; -vcs_vsn_cmd(VCS, Dir, Resources) when VCS =:= semver ; VCS =:= "semver" -> - vcs_vsn_cmd(git, Dir, Resources); -vcs_vsn_cmd({cmd, _Cmd}=Custom, _, _) -> +vcs_vsn_cmd(AppInfo, VCS, State) when VCS =:= semver ; VCS =:= "semver" -> + vcs_vsn_cmd(AppInfo, git, State); +vcs_vsn_cmd(_AppInfo, {cmd, _Cmd}=Custom, _) -> Custom; -vcs_vsn_cmd(VCS, Dir, Resources) when is_atom(VCS) -> - case rebar_resource:find_resource_module(VCS, Resources) of - {ok, Module} -> - Module:make_vsn(Dir); - {error, _} -> - unknown - end; -vcs_vsn_cmd(VCS, Dir, Resources) when is_list(VCS) -> +vcs_vsn_cmd(AppInfo, VCS, State) when is_atom(VCS) -> + rebar_resource_v2:make_vsn(AppInfo, VCS, State); +vcs_vsn_cmd(AppInfo, VCS, State) when is_list(VCS) -> try list_to_existing_atom(VCS) of AVCS -> - case vcs_vsn_cmd(AVCS, Dir, Resources) of + case vcs_vsn_cmd(AppInfo, AVCS, State) of unknown -> {plain, VCS}; Other -> Other end diff --git a/test/mock_git_resource.erl b/test/mock_git_resource.erl index 7b8279fc..fd6db573 100644 --- a/test/mock_git_resource.erl +++ b/test/mock_git_resource.erl @@ -46,8 +46,8 @@ unmock() -> mock_lock(_) -> meck:expect( ?MOD, lock, - fun(_AppDir, Git) -> - case Git of + fun(AppInfo, _) -> + case rebar_app_info:source(AppInfo) of {git, Url, {tag, Ref}} -> {git, Url, {ref, Ref}}; {git, Url, {ref, Ref}} -> {git, Url, {ref, Ref}}; {git, Url} -> {git, Url, {ref, "0.0.0"}}; @@ -62,7 +62,8 @@ mock_update(Opts) -> % ct:pal("TOUp: ~p", [ToUpdate]), meck:expect( ?MOD, needs_update, - fun(_Dir, {git, Url, _Ref}) -> + fun(AppInfo, _) -> + {git, Url, _Ref} = rebar_app_info:source(AppInfo), App = app(Url), % ct:pal("Needed update? ~p (~p) -> ~p", [App, {Url,_Ref}, lists:member(App, ToUpdate)]), lists:member(App, ToUpdate) @@ -78,7 +79,8 @@ mock_vsn(Opts) -> Default = proplists:get_value(default_vsn, Opts, "0.0.0"), meck:expect( ?MOD, make_vsn, - fun(Dir) -> + fun(AppInfo, _) -> + Dir = rebar_app_info:dir(AppInfo), case filelib:wildcard("*.app.src", filename:join([Dir,"src"])) of [AppSrc] -> {ok, App} = file:consult(AppSrc), @@ -108,7 +110,8 @@ mock_download(Opts, CreateType) -> Overrides = proplists:get_value(override_vsn, Opts, []), meck:expect( ?MOD, download, - fun (Dir, Git, _) -> + fun (Dir, AppInfo, _, _) -> + Git = rebar_app_info:source(AppInfo), filelib:ensure_dir(Dir), {git, Url, {_, Vsn}} = normalize_git(Git, Overrides, Default), App = app(Url), diff --git a/test/mock_pkg_resource.erl b/test/mock_pkg_resource.erl index 333eb621..7ec7441b 100644 --- a/test/mock_pkg_resource.erl +++ b/test/mock_pkg_resource.erl @@ -46,7 +46,10 @@ unmock() -> %% @doc creates values for a lock file. mock_lock(_) -> - meck:expect(?MOD, lock, fun(_AppDir, {pkg, Name, Vsn, Hash, _RepoConfig}) -> {pkg, Name, Vsn, Hash} end). + meck:expect(?MOD, lock, fun(AppInfo, _) -> + {pkg, Name, Vsn, Hash, _RepoConfig} = rebar_app_info:source(AppInfo), + {pkg, Name, Vsn, Hash} + end). %% @doc The config passed to the `mock/2' function can specify which apps %% should be updated on a per-name basis: `{update, ["App1", "App3"]}'. @@ -54,7 +57,8 @@ mock_update(Opts) -> ToUpdate = proplists:get_value(upgrade, Opts, []), meck:expect( ?MOD, needs_update, - fun(_Dir, {pkg, App, _Vsn, _Hash, _}) -> + fun(AppInfo, _) -> + {pkg, App, _Vsn, _Hash, _} = rebar_app_info:source(AppInfo), lists:member(binary_to_list(App), ToUpdate) end). @@ -62,7 +66,7 @@ mock_update(Opts) -> mock_vsn(_Opts) -> meck:expect( ?MOD, make_vsn, - fun(_Dir) -> + fun(_AppInfo, _) -> {error, "Replacing version of type pkg not supported."} end). @@ -79,19 +83,20 @@ mock_download(Opts) -> Config = proplists:get_value(config, Opts, []), meck:expect( ?MOD, download, - fun (Dir, {pkg, AppBin, Vsn, _, _}, _) -> - App = binary_to_list(AppBin), + fun (Dir, AppInfo, _, _) -> + {pkg, AppBin, Vsn, _, _} = rebar_app_info:source(AppInfo), + App = rebar_utils:to_list(AppBin), filelib:ensure_dir(Dir), AppDeps = proplists:get_value({App,Vsn}, Deps, []), - {ok, AppInfo} = rebar_test_utils:create_app( - Dir, App, binary_to_list(Vsn), + {ok, AppInfo1} = rebar_test_utils:create_app( + Dir, App, rebar_utils:to_list(Vsn), [kernel, stdlib] ++ [element(1,D) || D <- AppDeps] ), rebar_test_utils:create_config(Dir, [{deps, AppDeps}]++Config), - TarApp = App++"-"++binary_to_list(Vsn)++".tar", + TarApp = App++"-"++rebar_utils:to_list(Vsn)++".tar", Tarball = filename:join([Dir, TarApp]), Contents = filename:join([Dir, "contents.tar.gz"]), - Files = all_files(rebar_app_info:dir(AppInfo)), + Files = all_files(rebar_app_info:dir(AppInfo1)), ok = erl_tar:create(Contents, archive_names(Dir, App, Vsn, Files), [compressed]), diff --git a/test/rebar_compile_SUITE.erl b/test/rebar_compile_SUITE.erl index 6470b064..10effdae 100644 --- a/test/rebar_compile_SUITE.erl +++ b/test/rebar_compile_SUITE.erl @@ -1,69 +1,6 @@ -module(rebar_compile_SUITE). --export([suite/0, - init_per_suite/1, - end_per_suite/1, - init_per_testcase/2, - end_per_testcase/2, - init_per_group/2, - end_per_group/2, - all/0, - groups/0, - build_basic_app/1, paths_basic_app/1, clean_basic_app/1, - build_release_apps/1, paths_release_apps/1, clean_release_apps/1, - build_checkout_apps/1, paths_checkout_apps/1, - build_checkout_deps/1, paths_checkout_deps/1, - build_basic_srcdirs/1, paths_basic_srcdirs/1, - build_release_srcdirs/1, paths_release_srcdirs/1, - build_unbalanced_srcdirs/1, paths_unbalanced_srcdirs/1, - build_basic_extra_dirs/1, paths_basic_extra_dirs/1, clean_basic_extra_dirs/1, - build_release_extra_dirs/1, paths_release_extra_dirs/1, clean_release_extra_dirs/1, - build_unbalanced_extra_dirs/1, paths_unbalanced_extra_dirs/1, - build_extra_dirs_in_project_root/1, - paths_extra_dirs_in_project_root/1, - clean_extra_dirs_in_project_root/1, - recompile_when_hrl_changes/1, - recompile_when_included_hrl_changes/1, - recompile_when_opts_included_hrl_changes/1, - recompile_when_opts_change/1, - dont_recompile_when_opts_dont_change/1, - dont_recompile_yrl_or_xrl/1, - deps_in_path/1, - delete_beam_if_source_deleted/1, - checkout_priority/1, - highest_version_of_pkg_dep/1, - parse_transform_test/1, - erl_first_files_test/1, - mib_test/1, - umbrella_mib_first_test/1, - only_default_transitive_deps/1, - clean_all/1, - override_deps/1, - override_add_deps/1, - override_del_deps/1, - override_opts/1, - override_add_opts/1, - override_del_opts/1, - profile_deps/1, - profile_override_deps/1, - profile_override_add_deps/1, - profile_override_del_deps/1, - profile_override_opts/1, - profile_override_add_opts/1, - profile_override_del_opts/1, - deps_build_in_prod/1, - include_file_relative_to_working_directory/1, - include_file_in_src/1, - include_file_relative_to_working_directory_test/1, - include_file_in_src_test/1, - include_file_in_src_test_multiapp/1, - dont_recompile_when_erl_compiler_options_env_does_not_change/1, - recompile_when_erl_compiler_options_env_changes/1, - always_recompile_when_erl_compiler_options_set/1, - recompile_when_parse_transform_inline_changes/1, - recompile_when_parse_transform_as_opt_changes/1, - recursive/1,no_recursive/1, - regex_filter_skip/1, regex_filter_regression/1]). +-compile(export_all). -include_lib("common_test/include/ct.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -89,6 +26,7 @@ all() -> profile_deps, deps_build_in_prod, override_deps, override_add_deps, override_del_deps, override_opts, override_add_opts, override_del_opts, + apply_overrides_exactly_once, profile_override_deps, profile_override_add_deps, profile_override_del_deps, profile_override_opts, profile_override_add_opts, profile_override_del_opts, include_file_relative_to_working_directory, include_file_in_src, @@ -1405,6 +1343,35 @@ override_opts(Config) -> true = lists:member(compressed, proplists:get_value(options, Mod:module_info(compile), [])), false = lists:member(warn_missing_spec, proplists:get_value(options, Mod:module_info(compile), [])). +%% test for fix of https://github.com/erlang/rebar3/issues/1801 +%% only apply overrides once +%% verify by having an override add the macro TEST to the dep some_dep +%% building under `ct` will fail if the `add` is applied more than once +apply_overrides_exactly_once(Config) -> + AppDir = ?config(apps, Config), + + Deps = rebar_test_utils:expand_deps(git, [{"some_dep", "0.0.1", [{"other_dep", "0.0.1", []}]}]), + TopDeps = rebar_test_utils:top_level_deps(Deps), + + {SrcDeps, _} = rebar_test_utils:flat_deps(Deps), + mock_git_resource:mock([{deps, SrcDeps}]), + + Name = rebar_test_utils:create_random_name("app1_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + + RebarConfig = [{deps, TopDeps}, + {overrides, [ + {add, some_dep, [ + {erl_opts, [{d, 'TEST'}]} + ]} + ]}], + + rebar_test_utils:create_config(AppDir, RebarConfig), + + rebar_test_utils:run_and_check( + Config, RebarConfig, ["ct", "--compile_only"], {ok, [{app, Name}, {dep, "some_dep"}], "test"}). + override_add_opts(Config) -> AppDir = ?config(apps, Config), diff --git a/test/rebar_localfs_resource.erl b/test/rebar_localfs_resource.erl index 63687c00..3d1296a9 100644 --- a/test/rebar_localfs_resource.erl +++ b/test/rebar_localfs_resource.erl @@ -2,6 +2,7 @@ %% ex: ts=4 sw=4 et %% %% @doc A localfs custom resource (for testing purposes only) +%% implementing the deprecated rebar_resource instead of v2 %% %% ``` %% {deps, [ diff --git a/test/rebar_localfs_resource_v2.erl b/test/rebar_localfs_resource_v2.erl new file mode 100644 index 00000000..52af4d40 --- /dev/null +++ b/test/rebar_localfs_resource_v2.erl @@ -0,0 +1,50 @@ +%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ts=4 sw=4 et +%% +%% @doc A localfs custom resource (for testing purposes only) +%% +%% ``` +%% {deps, [ +%% %% Application files are copied from "/path/to/app_name" +%% {app_name, {localfs, "/path/to/app_name", undefined}} +%% ]}. +%% ''' +-module(rebar_localfs_resource_v2). + +-behaviour(rebar_resource_v2). + +-export([init/2 + ,lock/2 + ,download/4 + ,needs_update/2 + ,make_vsn/2]). + +-include_lib("eunit/include/eunit.hrl"). + +-spec init(atom(), rebar_state:t()) -> {ok, term()}. +init(Type, _State) -> + Resource = rebar_resource_v2:new(Type, ?MODULE, #{}), + {ok, Resource}. + +lock(AppInfo, _) -> + case rebar_app_info:source(AppInfo) of + {localfs, Path, _Ref} -> + {localfs, Path, undefined}; + {localfs, Path} -> + {localfs, Path, undefined} + end. + +needs_update(_AppInfo, _) -> + false. + +download(TmpDir, AppInfo, State, _) -> + download_(TmpDir, rebar_app_info:source(AppInfo), State). + +download_(TmpDir, {localfs, Path, _Ref}, State) -> + download_(TmpDir, {localfs, Path}, State); +download_(TmpDir, {localfs, Path}, _State) -> + ok = rebar_file_utils:cp_r(filelib:wildcard(Path ++ "/*"), TmpDir), + {ok, undefined}. + +make_vsn(_AppInfo, _) -> + {plain, "undefined"}. diff --git a/test/rebar_pkg_SUITE.erl b/test/rebar_pkg_SUITE.erl index d0545a4c..650117b7 100644 --- a/test/rebar_pkg_SUITE.erl +++ b/test/rebar_pkg_SUITE.erl @@ -118,7 +118,7 @@ good_uncached(Config) -> {Pkg,Vsn} = ?config(pkg, Config), State = ?config(state, Config), ?assertEqual({ok, true}, - rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum, #{}}, State)), + rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum, #{}}, State, #{}, true)), Cache = ?config(cache_dir, Config), ?assert(filelib:is_regular(filename:join(Cache, <>))). @@ -131,7 +131,7 @@ good_cached(Config) -> ?assert(filelib:is_regular(CachedFile)), {ok, Content} = file:read_file(CachedFile), ?assertEqual({ok, true}, - rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum, #{}}, State)), + rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum, #{}}, State, #{}, true)), {ok, Content} = file:read_file(CachedFile). badpkg(Config) -> @@ -143,7 +143,7 @@ badpkg(Config) -> ETagPath = filename:join(Cache, <>), rebar_pkg_resource:store_etag_in_cache(ETagPath, ?BADPKG_ETAG), ?assertMatch({bad_download, _Path}, - rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum, #{}}, State, false)), + rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum, #{}}, State, #{}, false)), %% The cached/etag files are there for forensic purposes ?assert(filelib:is_regular(ETagPath)), ?assert(filelib:is_regular(CachePath)). @@ -157,7 +157,7 @@ bad_to_good(Config) -> ?assert(filelib:is_regular(CachedFile)), {ok, Contents} = file:read_file(CachedFile), ?assertEqual({ok, true}, - rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum, #{}}, State)), + rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum, #{}}, State, #{}, true)), %% Cache has refreshed ?assert({ok, Contents} =/= file:read_file(CachedFile)). @@ -172,7 +172,7 @@ good_disconnect(Config) -> {ok, Content} = file:read_file(CachedFile), rebar_pkg_resource:store_etag_in_cache(ETagFile, ?BADPKG_ETAG), ?assertEqual({ok, true}, - rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum, #{}}, State)), + rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum, #{}}, State, #{}, true)), {ok, Content} = file:read_file(CachedFile). bad_disconnect(Config) -> @@ -180,7 +180,7 @@ bad_disconnect(Config) -> {Pkg,Vsn} = ?config(pkg, Config), State = ?config(state, Config), ?assertEqual({fetch_fail, Pkg, Vsn}, - rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum, #{}}, State)). + rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum, #{}}, State, #{}, true)). pkgs_provider(Config) -> Config1 = rebar_test_utils:init_rebar_state(Config), @@ -265,9 +265,9 @@ mock_config(Name, Config) -> meck:expect(rebar_state, resources, fun(_State) -> DefaultConfig = hex_core:default_config(), - [rebar_resource:new(pkg, rebar_pkg_resource, - #{repos => [DefaultConfig#{name => <<"hexpm">>}], - base_config => #{}})] + [rebar_resource_v2:new(pkg, rebar_pkg_resource, + #{repos => [DefaultConfig#{name => <<"hexpm">>}], + base_config => #{}})] end), meck:new(rebar_dir, [passthrough]), diff --git a/test/rebar_pkg_alias_SUITE.erl b/test/rebar_pkg_alias_SUITE.erl index 9b0b997d..8b2f1d9c 100644 --- a/test/rebar_pkg_alias_SUITE.erl +++ b/test/rebar_pkg_alias_SUITE.erl @@ -241,30 +241,16 @@ mock_config(Name, Config) -> end, AllDeps), - meck:expect(rebar_packages, registry_checksum, - fun(N, V, _, _) -> - case ets:match_object(Tid, {{N, V}, '_'}) of - [{{_, _}, [_, Checksum, _]}] -> - Checksum - %% _ -> - %% {ok, {200, #{}, #{releases => []}}} - end - end), - meck:new(hex_repo, [passthrough]), meck:expect(hex_repo, get_package, fun(_Config, PkgName) -> - case ets:match_object(Tid, {{PkgName,'_'}, '_'}) of - Matches -> - Releases = - [#{checksum => Checksum, - version => Vsn, - dependencies => [{DAppName, {pkg, DN, DV, undefined}} || {DN, DV, _, DAppName} <- Deps]} || - {{_, Vsn}, [Deps, Checksum, _]} <- Matches], - {ok, {200, #{}, #{releases => Releases}}}%% ; - %% _ -> - %% {ok, {200, #{}, #{releases => []}}} - end + Matches = ets:match_object(Tid, {{PkgName,'_'}, '_'}), + Releases = + [#{checksum => Checksum, + version => Vsn, + dependencies => [{DAppName, {pkg, DN, DV, undefined}} || {DN, DV, _, DAppName} <- Deps]} || + {{_, Vsn}, [Deps, Checksum, _]} <- Matches], + {ok, {200, #{}, #{releases => Releases}}} end), meck:expect(hex_repo, get_tarball, fun(_, _, _) -> diff --git a/test/rebar_pkg_repos_SUITE.erl b/test/rebar_pkg_repos_SUITE.erl index d7e1d06d..601566ec 100644 --- a/test/rebar_pkg_repos_SUITE.erl +++ b/test/rebar_pkg_repos_SUITE.erl @@ -200,15 +200,17 @@ auth_merging(_Config) -> <<"hexpm">> => #{write_key => <<"write key hexpm">>}}]} end), - ?assertMatch({ok, #{repos := [#{name := <<"repo-1">>, - read_key := <<"read key">>, - write_key := <<"write key">>}, - #{name := <<"repo-2">>, - read_key := <<"read key 2">>, - repos_key := <<"repos key 2">>, - write_key := <<"write key 2">>}, - #{name := <<"hexpm">>, - write_key := <<"write key hexpm">>}]}}, rebar_pkg_resource:init(State)), + ?assertMatch({ok, + #resource{state=#{repos := [#{name := <<"repo-1">>, + read_key := <<"read key">>, + write_key := <<"write key">>}, + #{name := <<"repo-2">>, + read_key := <<"read key 2">>, + repos_key := <<"repos key 2">>, + write_key := <<"write key 2">>}, + #{name := <<"hexpm">>, + write_key := <<"write key hexpm">>}]}}}, + rebar_pkg_resource:init(pkg, State)), ok. @@ -229,7 +231,8 @@ organization_merging(_Config) -> <<"hexpm">> => #{write_key => <<"write key hexpm">>}}]} end), - ?assertMatch({ok, #{repos := [#{name := <<"hexpm:repo-1">>, + ?assertMatch({ok, + #resource{state=#{repos := [#{name := <<"hexpm:repo-1">>, parent := <<"hexpm">>, read_key := <<"read key">>, write_key := <<"write key hexpm">>}, @@ -239,7 +242,8 @@ organization_merging(_Config) -> repos_key := <<"repos key 2">>, write_key := <<"write key 2">>}, #{name := <<"hexpm">>, - write_key := <<"write key hexpm">>}]}}, rebar_pkg_resource:init(State)), + write_key := <<"write key hexpm">>}]}}}, + rebar_pkg_resource:init(pkg, State)), ok. diff --git a/test/rebar_resource_SUITE.erl b/test/rebar_resource_SUITE.erl index 15f14dbb..ddacb91b 100644 --- a/test/rebar_resource_SUITE.erl +++ b/test/rebar_resource_SUITE.erl @@ -29,12 +29,15 @@ init_per_testcase(change_type_upgrade, Config) -> TypeStr = atom_to_list(Type), DirName = filename:join([?config(priv_dir, Config), "resource_"++TypeStr]), ec_file:mkdir_path(DirName), - [{path, DirName} | Config]. + + {ok, AppInfo} = rebar_app_info:new(test_app, "0.0.1", DirName), + AppInfo1 = rebar_app_info:source(AppInfo, ?config(resource, Config)), + + [{app, AppInfo1} | Config]. end_per_testcase(_, Config) -> Config. change_type_upgrade(Config) -> - ?assert(rebar_fetch:needs_update(?config(path, Config), - ?config(resource, Config), + ?assert(rebar_fetch:needs_update(?config(app, Config), ?config(state, Config))).