Merge remote-tracking branch 'upstream/master'

.travis.yml

@@ -22,6 +22,7 @@ script: "./bootstrap && ./rebar3 ct"
 branches:
   only:
   - master
+  - hex_core
 cache:
   directories:
   - "$HOME/.cache/rebar3/hex/default"

bootstrap

@@ -16,7 +16,10 @@ main(_) ->
                ,{getopt, []}
                ,{cf, []}
                ,{erlware_commons, ["ec_dictionary.erl", "ec_vsn.erl"]}
-               ,{certifi, ["certifi_pt.erl"]}],
+               ,{parse_trans, ["parse_trans.erl", "parse_trans_pp.erl",
+                               "parse_trans_codegen.erl"]}
+               ,{certifi, []}
+               ,{hex_core, []}],
     Deps = get_deps(),
     [fetch_and_compile(Dep, Deps) || Dep <- BaseDeps],
 
@@ -24,7 +27,7 @@ main(_) ->
     bootstrap_rebar3(),
 
     %% Build rebar.app from rebar.app.src
-    {ok, App} = rebar_app_info:new(rebar, "3.6.1", filename:absname("_build/default/lib/rebar/")),
+    {ok, App} = rebar_app_info:new(rebar, "3.7.0-rc1", filename:absname("_build/default/lib/rebar/")),
     rebar_otp_app:compile(rebar_state:new(), App),
 
     %% Because we are compiling files that are loaded already we want to silence
@@ -33,14 +36,6 @@ main(_) ->
 
     setup_env(),
     os:putenv("REBAR_PROFILE", "bootstrap"),
-    RegistryFile = default_registry_file(),
-    case filelib:is_file(RegistryFile) of
-        true ->
-            ok;
-        false ->
-            rebar3:run(["update"])
-    end,
-
     {ok, State} = rebar3:run(["compile"]),
     reset_env(),
     os:putenv("REBAR_PROFILE", ""),
@@ -54,11 +49,6 @@ main(_) ->
     %% Done with compile, can turn back on error logger
     error_logger:tty(true).
 
-default_registry_file() ->
-    {ok, [[Home]]} = init:get_argument(home),
-    CacheDir = filename:join([Home, ".cache", "rebar3"]),
-    filename:join([CacheDir, "hex", "default", "registry"]).
-
 fetch_and_compile({Name, ErlFirstFiles}, Deps) ->
     case lists:keyfind(Name, 1, Deps) of
         {Name, Vsn} ->
@@ -171,7 +161,7 @@ bootstrap_rebar3() ->
     Res = symlink_or_copy(filename:absname("src"),
                           filename:absname("_build/default/lib/rebar/src")),
     true = Res == ok orelse Res == exists,
-    Sources = ["src/rebar_resource.erl" | filelib:wildcard("src/*.erl")],
+    Sources = ["src/rebar_resource_v2.erl", "src/rebar_resource.erl" | filelib:wildcard("src/*.erl")],
     [compile_file(X, [{outdir, "_build/default/lib/rebar/ebin/"}
                      ,return | additional_defines()]) || X <- Sources],
     code:add_patha(filename:absname("_build/default/lib/rebar/ebin")).

priv/templates/otp_app.app.src

@@ -1,8 +1,9 @@
-{application, {{name}},
- [{description, "{{desc}}"},
+{{=@@ @@=}}
+{application, @@name@@,
+ [{description, "@@desc@@"},
   {vsn, "0.1.0"},
   {registered, []},
-  {mod, { {{name}}_app, []}},
+  {mod, {@@name@@_app, []}},
   {applications,
    [kernel,
     stdlib

priv/templates/plugin_README.md

@@ -1,7 +1,8 @@
-{{name}}
+{{=@@ @@=}}
+@@name@@
 =====
 
-{{desc}}
+@@desc@@
 
 Build
 -----
@@ -14,13 +15,13 @@ Use
 Add the plugin to your rebar config:
 
     {plugins, [
-        { {{name}}, ".*", {git, "git@host:user/{{name}}.git", {tag, "0.1.0"}}}
+        {@@name@@, {git, "https://host/user/@@name@@.git", {tag, "0.1.0"}}}
     ]}.
 
 Then just call your plugin directly in an existing application:
 
 
-    $ rebar3 {{name}}
-    ===> Fetching {{name}}
-    ===> Compiling {{name}}
+    $ rebar3 @@name@@
+    ===> Fetching @@name@@
+    ===> Compiling @@name@@
     <Plugin Output>

priv/templates/relx_rebar.config

@@ -1,8 +1,9 @@
+{{=@@ @@=}}
 {erl_opts, [debug_info]}.
 {deps, []}.
 
-{relx, [{release, { {{name}}, "0.1.0" },
-         [{{name}},
+{relx, [{release, {@@name@@, "0.1.0"},
+         [@@name@@,
           sasl]},
 
         {sys_config, "./config/sys.config"},

priv/templates/sup.erl

@@ -1,9 +1,10 @@
+{{=@@ @@=}}
 %%%-------------------------------------------------------------------
-%% @doc {{name}} top level supervisor.
+%% @doc @@name@@ top level supervisor.
 %% @end
 %%%-------------------------------------------------------------------
 
--module({{name}}_sup).
+-module(@@name@@_sup).
 
 -behaviour(supervisor).
 
@@ -31,7 +32,7 @@ start_link() ->
 %% Before OTP 18 tuples must be used to specify a child. e.g.
 %% Child :: {Id,StartFunc,Restart,Shutdown,Type,Modules}
 init([]) ->
-    {ok, { {one_for_all, 0, 1}, []} }.
+    {ok, {{one_for_all, 0, 1}, []}}.
 
 %%====================================================================
 %% Internal functions

priv/templates/sys.config

@@ -1,3 +1,4 @@
+{{=@@ @@=}}
 [
-  { {{name}}, []}
+  {@@name@@, []}
 ].

rebar.config

@@ -1,15 +1,17 @@
 %% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*-
 %% ex: ts=4 sw=4 ft=erlang et
 
-{deps, [{erlware_commons,     "1.2.0"},
+{deps, [{erlware_commons,     "1.3.0"},
         {ssl_verify_fun,      "1.1.3"},
-        {certifi,             "2.0.0"},
+        {certifi,             "2.3.1"},
+        {parse_trans,         "3.3.0"}, % force otp-21 compat
         {providers,           "1.7.0"},
         {getopt,              "1.0.1"},
-        {bbmustache,          "1.5.0"},
-        {relx,                "3.26.0"},
+        {bbmustache,          "1.6.0"},
+        {relx,                "3.27.0"},
         {cf,                  "0.2.2"},
         {cth_readable,        "1.4.2"},
+        {hex_core,            "0.2.0"},
         {eunit_formatters,    "0.5.0"}]}.
 
 {post_hooks, [{"(linux|darwin|solaris|freebsd|netbsd|openbsd)",
@@ -31,8 +33,10 @@
 {erl_opts, [{platform_define, "^[0-9]+", namespaced_types},
             {platform_define, "^(19|2)", rand_only},
             {platform_define, "^2", unicode_str},
+            {platform_define, "^2", filelib_find_source},
             {platform_define, "^(R|1|20)", fun_stacktrace},
-            warnings_as_errors]}.
+            warnings_as_errors
+           ]}.
 
 %% Use OTP 18+ when dialyzing rebar3
 {dialyzer, [
@@ -42,7 +46,7 @@
 
 %% Profiles
 {profiles, [{test, [
-                   {deps, [{meck, "0.8.7"}]},
+                   {deps, [{meck, "0.8.12"}]},
                    {erl_opts, [debug_info, nowarn_export_all]}
                    ]
             },

rebar.config.sample

@@ -144,7 +144,7 @@
 %% == Escript ==
 
 %% name of the main OTP application to boot
-{escript_main_app, application}
+{escript_main_app, application}.
 %% Name of the resulting escript executable
 {escript_name, "application"}.
 %% apps (other than main and deps) to be included
@@ -158,7 +158,7 @@
 %% == EUnit ==
 
 %% eunit:test(Tests)
-{eunit_tests, [{application, rebar3}]}
+{eunit_tests, [{application, rebar3}]}.
 %% Options for eunit:test(Tests, Opts)
 {eunit_opts, [verbose]}.
 %% Additional compile options for eunit. erl_opts is also used
@@ -198,7 +198,7 @@
 %% Only clean, ct, compile, eunit, release, and tar can be hooked around
 
 %% runs 'clean' before 'compile'
-{provider_hooks, [{pre, [{compile, clean}]}]}
+{provider_hooks, [{pre, [{compile, clean}]}]}.
 
 
 %% == Releases ==

rebar.lock

@@ -1,24 +1,28 @@
 {"1.1.0",
-[{<<"bbmustache">>,{pkg,<<"bbmustache">>,<<"1.5.0">>},0},
- {<<"certifi">>,{pkg,<<"certifi">>,<<"2.0.0">>},0},
+[{<<"bbmustache">>,{pkg,<<"bbmustache">>,<<"1.6.0">>},0},
+ {<<"certifi">>,{pkg,<<"certifi">>,<<"2.3.1">>},0},
  {<<"cf">>,{pkg,<<"cf">>,<<"0.2.2">>},0},
  {<<"cth_readable">>,{pkg,<<"cth_readable">>,<<"1.4.2">>},0},
- {<<"erlware_commons">>,{pkg,<<"erlware_commons">>,<<"1.2.0">>},0},
+ {<<"erlware_commons">>,{pkg,<<"erlware_commons">>,<<"1.3.0">>},0},
  {<<"eunit_formatters">>,{pkg,<<"eunit_formatters">>,<<"0.5.0">>},0},
  {<<"getopt">>,{pkg,<<"getopt">>,<<"1.0.1">>},0},
+ {<<"hex_core">>,{pkg,<<"hex_core">>,<<"0.2.0">>},0},
+ {<<"parse_trans">>,{pkg,<<"parse_trans">>,<<"3.3.0">>},0},
  {<<"providers">>,{pkg,<<"providers">>,<<"1.7.0">>},0},
- {<<"relx">>,{pkg,<<"relx">>,<<"3.26.0">>},0},
+ {<<"relx">>,{pkg,<<"relx">>,<<"3.27.0">>},0},
  {<<"ssl_verify_fun">>,{pkg,<<"ssl_verify_fun">>,<<"1.1.3">>},0}]}.
 [
 {pkg_hash,[
- {<<"bbmustache">>, <<"8CFDE0602E90A4057E161BF5288ADE854B4E511E2E8924966A8438730E958381">>},
- {<<"certifi">>, <<"A0C0E475107135F76B8C1D5BC7EFB33CD3815CB3CF3DEA7AEFDD174DABEAD064">>},
+ {<<"bbmustache">>, <<"7AC372AEC621A69C369DF237FBD9986CAABCDD6341089FE5F42E5A7A4AC706B8">>},
+ {<<"certifi">>, <<"D0F424232390BF47D82DA8478022301C561CF6445B5B5FB6A84D49A9E76D2639">>},
  {<<"cf">>, <<"7F2913FFF90ABCABD0F489896CFEB0B0674F6C8DF6C10B17A83175448029896C">>},
  {<<"cth_readable">>, <<"0F57B4EB7DA7F5438F422312245F9143A1B3118C11B6BAE5C3D1391C9EE88322">>},
- {<<"erlware_commons">>, <<"2BAB99CF88941145767A502F1209886F1F0D31695EEF21978A30F15E645721E0">>},
+ {<<"erlware_commons">>, <<"1705CF2AB4212EF235C21971A55E22E2A39055C05B9C65C8848126865F42A07A">>},
  {<<"eunit_formatters">>, <<"6A9133943D36A465D804C1C5B6E6839030434B8879C5600D7DDB5B3BAD4CCB59">>},
  {<<"getopt">>, <<"C73A9FA687B217F2FF79F68A3B637711BB1936E712B521D8CE466B29CBF7808A">>},
+ {<<"hex_core">>, <<"3A7EACCFB8ADD3FF05D950C10ED5BDB5D0C48C988EBBC5D7AE2A55498F0EFF1B">>},
+ {<<"parse_trans">>, <<"09765507A3C7590A784615CFD421D101AEC25098D50B89D7AA1D66646BC571C1">>},
  {<<"providers">>, <<"BBF730563914328EC2511D205E6477A94831DB7297DE313B3872A2B26C562EAB">>},
- {<<"relx">>, <<"DD645ECAA1AB1647DB80D3E9BCAE0B39ED0A536EF37245F6A74B114C6D0F4E87">>},
+ {<<"relx">>, <<"96CC7663EDCC02A8117AB0C64FE6D15BE79760C08726ABEAD1DAACE11BFBF75D">>},
  {<<"ssl_verify_fun">>, <<"6C49665D4326E26CD4A5B7BD54AA442B33DADFB7C5D59A0D0CD0BF5534BBFBD7">>}]}
 ].

src/rebar.app.src

@@ -30,6 +30,7 @@
                   relx,
                   cf,
                   inets,
+                  hex_core,
                   eunit_formatters]},
   {env, [
         %% Default log level
@@ -39,6 +40,9 @@
                      {pkg, rebar_pkg_resource},
                      {hg, rebar_hg_resource}]},
 
+        {compilers, [rebar_compiler_xrl, rebar_compiler_yrl,
+                     rebar_compiler_mib, rebar_compiler_erl]},
+
         {providers, [rebar_prv_app_discovery,
                      rebar_prv_as,
                      rebar_prv_bare_compile,
@@ -67,6 +71,7 @@
                      rebar_prv_release,
                      rebar_prv_relup,
                      rebar_prv_report,
+                     rebar_prv_repos,
                      rebar_prv_shell,
                      rebar_prv_state,
                      rebar_prv_tar,

src/rebar.hrl

@@ -25,14 +25,35 @@
 -define(CONFIG_VERSION, "1.1.0").
 -define(DEFAULT_CDN, "https://repo.hex.pm/").
 -define(REMOTE_PACKAGE_DIR, "tarballs").
--define(REMOTE_REGISTRY_FILE, "registry.ets.gz").
 -define(LOCK_FILE, "rebar.lock").
 -define(DEFAULT_COMPILER_SOURCE_FORMAT, relative).
-
--define(PACKAGE_INDEX_VERSION, 3).
+-define(PACKAGE_INDEX_VERSION, 5).
 -define(PACKAGE_TABLE, package_index).
 -define(INDEX_FILE, "packages.idx").
--define(REGISTRY_FILE, "registry").
+-define(HEX_AUTH_FILE, "hex.config").
+-define(PUBLIC_HEX_REPO, <<"hexpm">>).
+
+%% ignore this function in all modules
+%% not every module that exports it and relies on it being called implements provider
+-ignore_xref([{format_error, 1}]).
+
+%% the package record is used in a select match spec which upsets dialyzer
+%% this is the suggested workaround from Tobias
+%% http://erlang.org/pipermail/erlang-questions/2009-February/041445.html
+-type ms_field() :: '$1' | '_'.
+
+%% TODO: change package and requirement keys to be required (:=) after dropping support for OTP-18
+-record(package, {key :: {unicode:unicode_binary() | ms_field(), unicode:unicode_binary() | ms_field(),
+                          unicode:unicode_binary() | ms_field()},
+                  checksum :: binary() | ms_field(),
+                  retired :: boolean() | ms_field(),
+                  dependencies :: [#{package => unicode:unicode_binary(),
+                                     requirement => unicode:unicode_binary()}] | ms_field()}).
+
+-record(resource, {type :: atom(),
+                   module :: module(),
+                   state :: term(),
+                   implementation :: rebar_resource | rebar_resource_v2}).
 
 -ifdef(namespaced_types).
 -type rebar_dict() :: dict:dict().

src/rebar3.erl

@@ -103,7 +103,7 @@ run(RawArgs) ->
     case erlang:system_info(version) of
         "6.1" ->
             ?WARN("Due to a filelib bug in Erlang 17.1 it is recommended"
-                 "you update to a newer release.", []);
+                  "you update to a newer release.", []);
         _ ->
             ok
     end,
@@ -139,8 +139,17 @@ run_aux(State, RawArgs) ->
                      rebar_state:set(State1, rebar_packages_cdn, CDN)
              end,
 
+    Compilers = application:get_env(rebar, compilers, []),
+    State0 = rebar_state:compilers(State2, Compilers),
+
+    %% TODO: this means use of REBAR_PROFILE=profile will replace the repos with
+    %% the repos defined in the profile. But it will not work with `as profile`.
+    %% Maybe it shouldn't work with either to be consistent?
+    Resources = application:get_env(rebar, resources, []),
+    State2_ = rebar_state:create_resources(Resources, State0),
+
     %% bootstrap test profile
-    State3 = rebar_state:add_to_profile(State2, test, test_state(State1)),
+    State3 = rebar_state:add_to_profile(State2_, test, test_state(State1)),
 
     %% Process each command, resetting any state between each one
     BaseDir = rebar_state:get(State, base_dir, ?DEFAULT_BASE_DIR),
@@ -166,7 +175,20 @@ run_aux(State, RawArgs) ->
 
     State10 = rebar_state:code_paths(State9, default, code:get_path()),
 
-    rebar_core:init_command(rebar_state:command_args(State10, Args), Task).
+    case rebar_core:init_command(rebar_state:command_args(State10, Args), Task) of
+        {ok, State11} ->
+            case rebar_state:get(State11, caller, command_line) of
+                api ->
+                    rebar_paths:unset_paths([deps, plugins], State11),
+                    {ok, State11};
+                _ ->
+                    {ok, State11}
+            end;
+        Other ->
+            Other
+    end.
+
+
 
 %% @doc set up base configuration having to do with verbosity, where
 %% to find config files, and so on, and return an internal rebar3 state term.
@@ -375,7 +397,11 @@ state_from_global_config(Config, GlobalConfigFile) ->
 
     %% We don't want to worry about global plugin install state effecting later
     %% usage. So we throw away the global profile state used for plugin install.
-    GlobalConfigThrowAway = rebar_state:current_profiles(GlobalConfig, [global]),
+    GlobalConfigThrowAway0 = rebar_state:current_profiles(GlobalConfig, [global]),
+
+    Resources = application:get_env(rebar, resources, []),
+    GlobalConfigThrowAway = rebar_state:create_resources(Resources, GlobalConfigThrowAway0),
+
     GlobalState = case rebar_state:get(GlobalConfigThrowAway, plugins, []) of
                       [] ->
                           GlobalConfigThrowAway;
@@ -386,7 +412,8 @@ state_from_global_config(Config, GlobalConfigFile) ->
                   end,
     GlobalPlugins = rebar_state:providers(GlobalState),
     GlobalConfig2 = rebar_state:set(GlobalConfig, plugins, []),
-    GlobalConfig3 = rebar_state:set(GlobalConfig2, {plugins, global}, rebar_state:get(GlobalConfigThrowAway, plugins, [])),
+    GlobalConfig3 = rebar_state:set(GlobalConfig2, {plugins, global},
+                                    rebar_state:get(GlobalConfigThrowAway, plugins, [])),
     rebar_state:providers(rebar_state:new(GlobalConfig3, Config), GlobalPlugins).
 
 -spec test_state(rebar_state:t()) -> [{'extra_src_dirs',[string()]} | {'erl_opts',[any()]}].

src/rebar_api.erl

@@ -9,6 +9,8 @@
          expand_env_variable/3,
          get_arch/0,
          wordsize/0,
+         set_paths/2,
+         unset_paths/2,
          add_deps_to_path/1,
          restore_code_path/1,
          processing_base_dir/1,
@@ -67,6 +69,21 @@ get_arch() ->
 wordsize() ->
     rebar_utils:wordsize().
 
+%% @doc Set code paths. Takes arguments of the form
+%% `[plugins, deps]' or `[deps, plugins]' and ensures the
+%% project's app and dependencies are set in the right order
+%% for the next bit of execution
+-spec set_paths(rebar_paths:targets(), rebar_state:t()) -> ok.
+set_paths(List, State) ->
+    rebar_paths:set_paths(List, State).
+
+%% @doc Unsets code paths. Takes arguments of the form
+%% `[plugins, deps]' or `[deps, plugins]' and ensures the
+%% paths are no longer active.
+-spec unset_paths(rebar_paths:targets(), rebar_state:t()) -> ok.
+unset_paths(List, State) ->
+    rebar_paths:unset_paths(List, State).
+
 %% @doc Add deps to the code path
 -spec add_deps_to_path(rebar_state:t()) -> ok.
 add_deps_to_path(State) ->
@@ -88,4 +105,4 @@ processing_base_dir(State) ->
 %% its configuration, including for validation of certs.
 -spec ssl_opts(string()) -> [term()].
 ssl_opts(Url) ->
-    rebar_pkg_resource:ssl_opts(Url).
+    rebar_utils:ssl_opts(Url).

src/rebar_app_discover.erl

@@ -7,10 +7,9 @@
          find_unbuilt_apps/1,
          find_apps/1,
          find_apps/2,
-         find_apps/3,
+         find_apps/4,
          find_app/2,
-         find_app/3,
-         find_app/4]).
+         find_app/3]).
 
 -include("rebar.hrl").
 -include_lib("providers/include/providers.hrl").
@@ -24,7 +23,7 @@ do(State, LibDirs) ->
     Dirs = [filename:join(BaseDir, LibDir) || LibDir <- LibDirs],
     RebarOpts = rebar_state:opts(State),
     SrcDirs = rebar_dir:src_dirs(RebarOpts, ["src"]),
-    Apps = find_apps(Dirs, SrcDirs, all),
+    Apps = find_apps(Dirs, SrcDirs, all, State),
     ProjectDeps = rebar_state:deps_names(State),
     DepsDir = rebar_dir:deps_dir(State),
     CurrentProfiles = rebar_state:current_profiles(State),
@@ -53,7 +52,7 @@ do(State, LibDirs) ->
                         Name = rebar_app_info:name(AppInfo),
                         case enable(State, AppInfo) of
                             true ->
-                                {AppInfo1, StateAcc1} = merge_deps(AppInfo, StateAcc),
+                                {AppInfo1, StateAcc1} = merge_opts(AppInfo, StateAcc),
                                 OutDir = filename:join(DepsDir, Name),
                                 AppInfo2 = rebar_app_info:out_dir(AppInfo1, OutDir),
                                 ProjectDeps1 = lists:delete(Name, ProjectDeps),
@@ -88,34 +87,34 @@ format_error({module_list, File}) ->
 format_error({missing_module, Module}) ->
     io_lib:format("Module defined in app file missing: ~p~n", [Module]).
 
-%% @doc handles the merging and application of profiles and overrides
-%% for a given application, within its own context.
--spec merge_deps(rebar_app_info:t(), rebar_state:t()) ->
+%% @doc merges configuration of a project app and the top level state
+%% some configuration like erl_opts must be merged into a subapp's opts
+%% while plugins and hooks need to be kept defined to only either the
+%% top level state or an individual application.
+-spec merge_opts(rebar_app_info:t(), rebar_state:t()) ->
     {rebar_app_info:t(), rebar_state:t()}.
-merge_deps(AppInfo, State) ->
+merge_opts(AppInfo, State) ->
     %% These steps make sure that hooks and artifacts are run in the context of
     %% the application they are defined at. If an umbrella structure is used and
-    %% they are deifned at the top level they will instead run in the context of
+    %% they are defined at the top level they will instead run in the context of
     %% the State and at the top level, not as part of an application.
     CurrentProfiles = rebar_state:current_profiles(State),
-    Default = reset_hooks(rebar_state:default(State), CurrentProfiles),
-    {C, State1} = project_app_config(AppInfo, State),
-    AppInfo0 = rebar_app_info:update_opts(AppInfo, Default, C),
+    {AppInfo1, State1} = maybe_reset_hooks_plugins(AppInfo, State),
 
-    Name = rebar_app_info:name(AppInfo0),
+    Name = rebar_app_info:name(AppInfo1),
 
     %% We reset the opts here to default so no profiles are applied multiple times
-    AppInfo1 = rebar_app_info:apply_overrides(rebar_state:get(State1, overrides, []), AppInfo0),
-    AppInfo2 = rebar_app_info:apply_profiles(AppInfo1, CurrentProfiles),
+    AppInfo2 = rebar_app_info:apply_overrides(rebar_state:get(State1, overrides, []), AppInfo1),
+    AppInfo3 = rebar_app_info:apply_profiles(AppInfo2, CurrentProfiles),
 
     %% Will throw an exception if checks fail
-    rebar_app_info:verify_otp_vsn(AppInfo2),
+    rebar_app_info:verify_otp_vsn(AppInfo3),
 
     State2 = lists:foldl(fun(Profile, StateAcc) ->
-                                 handle_profile(Profile, Name, AppInfo2, StateAcc)
+                                 handle_profile(Profile, Name, AppInfo3, StateAcc)
                          end, State1, lists:reverse(CurrentProfiles)),
 
-    {AppInfo2, State2}.
+    {AppInfo3, State2}.
 
 %% @doc Applies a given profile for an app, ensuring the deps
 %% match the context it will require.
@@ -153,30 +152,32 @@ parse_profile_deps(Profile, Name, Deps, Opts, State) ->
                               ,Locks
                               ,1).
 
-%% @doc Find the app-level config and return the state updated
-%% with the relevant app-level data.
--spec project_app_config(rebar_app_info:t(), rebar_state:t()) ->
-    {Config, rebar_state:t()} when
-      Config :: [any()].
-project_app_config(AppInfo, State) ->
-    C = rebar_config:consult(rebar_app_info:dir(AppInfo)),
+%% reset the State hooks if there is a top level application
+-spec maybe_reset_hooks_plugins(AppInfo, State) ->  {AppInfo, State} when
+      AppInfo :: rebar_app_info:t(),
+      State :: rebar_state:t().
+maybe_reset_hooks_plugins(AppInfo, State) ->
     Dir = rebar_app_info:dir(AppInfo),
-    Opts = maybe_reset_hooks(Dir, rebar_state:opts(State), State),
-    {C, rebar_state:opts(State, Opts)}.
-
-%% @private Check if the app is at the root of the project.
-%% If it is, then drop the hooks from the config so they aren't run twice
--spec maybe_reset_hooks(file:filename(), Opts, rebar_state:t()) -> Opts when
-      Opts :: rebar_dict().
-maybe_reset_hooks(Dir, Opts, State) ->
+    CurrentProfiles = rebar_state:current_profiles(State),
     case ec_file:real_dir_path(rebar_dir:root_dir(State)) of
         Dir ->
-            CurrentProfiles = rebar_state:current_profiles(State),
-            reset_hooks(Opts, CurrentProfiles);
+            Opts = reset_hooks(rebar_state:opts(State), CurrentProfiles),
+            State1 = rebar_state:opts(State, Opts),
+
+            %% set plugins to empty since this is an app at the top level
+            %% and top level plugins are installed in run_aux
+            AppInfo1 = rebar_app_info:set(rebar_app_info:set(AppInfo, {plugins,default}, []), plugins, []),
+
+            {AppInfo1, State1};
         _ ->
-            Opts
+            %% if not in the top root directory then we need to merge in the
+            %% default state opts to this subapp's opts
+            Default = reset_hooks(rebar_state:default(State), CurrentProfiles),
+            AppInfo1 = rebar_app_info:update_opts(AppInfo, Default),
+            {AppInfo1, State}
     end.
 
+
 %% @doc make the hooks empty for a given set of options
 -spec reset_hooks(Opts, Profiles) ->
     Opts when
@@ -205,8 +206,8 @@ reset_hooks(Opts, CurrentProfiles) ->
 -spec all_app_dirs([file:name()]) -> [{file:name(), [file:name()]}].
 all_app_dirs(LibDirs) ->
     lists:flatmap(fun(LibDir) ->
-                        SrcDirs = find_config_src(LibDir, ["src"]),
-                        app_dirs(LibDir, SrcDirs)
+                          {_, SrcDirs} = find_config_src(LibDir, ["src"]),
+                          app_dirs(LibDir, SrcDirs)
                   end, LibDirs).
 
 %% @private find the directories for all apps based on their source dirs
@@ -264,11 +265,11 @@ find_apps(LibDirs, Validate) ->
 %% @doc for each directory passed, with the configured source directories,
 %% find all apps according to the validity rule passed in.
 %% Returns all the related app info records.
--spec find_apps([file:filename_all()], [file:filename_all()], valid | invalid | all) -> [rebar_app_info:t()].
-find_apps(LibDirs, SrcDirs, Validate) ->
+-spec find_apps([file:filename_all()], [file:filename_all()], valid | invalid | all, rebar_state:t()) -> [rebar_app_info:t()].
+find_apps(LibDirs, SrcDirs, Validate, State) ->
     rebar_utils:filtermap(
       fun({AppDir, AppSrcDirs}) ->
-            find_app(rebar_app_info:new(), AppDir, AppSrcDirs, Validate)
+            find_app(rebar_app_info:new(), AppDir, AppSrcDirs, Validate, State)
       end,
       all_app_dirs(LibDirs, SrcDirs)
     ).
@@ -278,8 +279,9 @@ find_apps(LibDirs, SrcDirs, Validate) ->
 %% app info record.
 -spec find_app(file:filename_all(), valid | invalid | all) -> {true, rebar_app_info:t()} | false.
 find_app(AppDir, Validate) ->
-    SrcDirs = find_config_src(AppDir, ["src"]),
-    find_app(rebar_app_info:new(), AppDir, SrcDirs, Validate).
+    {Config, SrcDirs} = find_config_src(AppDir, ["src"]),
+    AppInfo = rebar_app_info:update_opts(rebar_app_info:new(), dict:new(), Config),
+    find_app_(AppInfo, AppDir, SrcDirs, Validate).
 
 %% @doc check that a given app in a directory is there, and whether it's
 %% valid or not based on the second argument. Returns the related
@@ -291,16 +293,35 @@ find_app(AppInfo, AppDir, Validate) ->
     %% of src/
     AppOpts = rebar_app_info:opts(AppInfo),
     SrcDirs = rebar_dir:src_dirs(AppOpts, ["src"]),
-    find_app(AppInfo, AppDir, SrcDirs, Validate).
+    find_app_(AppInfo, AppDir, SrcDirs, Validate).
 
 %% @doc check that a given app in a directory is there, and whether it's
 %% valid or not based on the second argument. The third argument includes
 %% the directories where source files can be located. Returns the related
 %% app info record.
 -spec find_app(rebar_app_info:t(), file:filename_all(),
-               [file:filename_all()], valid | invalid | all) ->
+               [file:filename_all()], valid | invalid | all, rebar_state:t()) ->
     {true, rebar_app_info:t()} | false.
+find_app(AppInfo, AppDir, SrcDirs, Validate, State) ->
+    AppInfo1 = case ec_file:real_dir_path(rebar_dir:root_dir(State)) of
+                   AppDir ->
+                       Opts = rebar_state:opts(State),
+                       rebar_app_info:default(rebar_app_info:opts(AppInfo, Opts), Opts);
+                   _ ->
+                       Config = rebar_config:consult(AppDir),
+                       rebar_app_info:update_opts(AppInfo, rebar_app_info:opts(AppInfo), Config)
+               end,
+    find_app_(AppInfo1, AppDir, SrcDirs, Validate).
+
 find_app(AppInfo, AppDir, SrcDirs, Validate) ->
+    Config = rebar_config:consult(AppDir),
+    AppInfo1 = rebar_app_info:update_opts(AppInfo, rebar_app_info:opts(AppInfo), Config),
+    find_app_(AppInfo1, AppDir, SrcDirs, Validate).
+
+-spec find_app_(rebar_app_info:t(), file:filename_all(),
+               [file:filename_all()], valid | invalid | all) ->
+    {true, rebar_app_info:t()} | false.
+find_app_(AppInfo, AppDir, SrcDirs, Validate) ->
     AppFile = filelib:wildcard(filename:join([AppDir, "ebin", "*.app"])),
     AppSrcFile = lists:append(
         [filelib:wildcard(filename:join([AppDir, SrcDir, "*.app.src"]))
@@ -331,17 +352,14 @@ create_app_info(AppInfo, AppDir, AppFile) ->
     AppInfo2 = rebar_app_info:applications(
                  rebar_app_info:app_details(AppInfo1, AppDetails),
                  IncludedApplications++Applications),
-    C = rebar_config:consult(AppDir),
-    AppInfo3 = rebar_app_info:update_opts(AppInfo2,
-                                          rebar_app_info:opts(AppInfo2), C),
-    Valid = case rebar_app_utils:validate_application_info(AppInfo3) =:= true
-                andalso rebar_app_info:has_all_artifacts(AppInfo3) =:= true of
+    Valid = case rebar_app_utils:validate_application_info(AppInfo2) =:= true
+                andalso rebar_app_info:has_all_artifacts(AppInfo2) =:= true of
                 true ->
                     true;
                 _ ->
                     false
             end,
-    rebar_app_info:dir(rebar_app_info:valid(AppInfo3, Valid), AppDir).
+    rebar_app_info:dir(rebar_app_info:valid(AppInfo2, Valid), AppDir).
 
 %% @doc Read in and parse the .app file if it is availabe. Do the same for
 %% the .app.src file if it exists.
@@ -403,12 +421,20 @@ try_handle_app_file(_AppInfo, Other, _AppDir, _AppSrcFile, _, _Validate) ->
       AppFile :: file:filename(),
       AppDir :: file:filename(),
       AppSrcFile :: file:filename().
-try_handle_app_src_file(_AppInfo, _, _AppDir, [], _Validate) ->
-    false;
+try_handle_app_src_file(AppInfo, _, _AppDir, [], _Validate) ->
+    %% if .app and .app.src are not found check for a mix config file
+    %% it is assumed a plugin will build the application, including
+    %% a .app after this step
+    case filelib:is_file(filename:join(rebar_app_info:dir(AppInfo), "mix.exs")) of
+        true ->
+            {true, rebar_app_info:project_type(AppInfo, mix)};
+        false ->
+            false
+    end;
 try_handle_app_src_file(_AppInfo, _, _AppDir, _AppSrcFile, valid) ->
     false;
 try_handle_app_src_file(AppInfo, _, AppDir, [File], Validate) when Validate =:= invalid
-                                                                 ; Validate =:= all ->
+                                                                   ; Validate =:= all ->
     AppInfo1 = rebar_app_info:app_file(AppInfo, undefined),
     AppInfo2 = create_app_info(AppInfo1, AppDir, File),
     case filename:extension(File) of
@@ -437,8 +463,8 @@ to_atom(Bin) ->
 find_config_src(AppDir, Default) ->
     case rebar_config:consult(AppDir) of
         [] ->
-            Default;
+            {[], Default};
         Terms ->
             %% TODO: handle profiles I guess, but we don't have that info
-            proplists:get_value(src_dirs, Terms, Default)
+            {Terms, proplists:get_value(src_dirs, Terms, Default)}
     end.

src/rebar_app_info.erl

@@ -7,6 +7,8 @@
          new/4,
          new/5,
          update_opts/3,
+         update_opts/2,
+         update_opts_deps/2,
          discover/1,
          name/1,
          name/2,
@@ -22,7 +24,6 @@
          parent/2,
          original_vsn/1,
          original_vsn/2,
-         ebin_dir/1,
          priv_dir/1,
          applications/1,
          applications/2,
@@ -36,6 +37,8 @@
          dir/2,
          out_dir/1,
          out_dir/2,
+         ebin_dir/1,
+         ebin_dir/2,
          default/1,
          default/2,
          opts/1,
@@ -43,16 +46,18 @@
          get/2,
          get/3,
          set/3,
-         resource_type/1,
-         resource_type/2,
          source/1,
          source/2,
+         project_type/1,
+         project_type/2,
          is_lock/1,
          is_lock/2,
          is_checkout/1,
          is_checkout/2,
          valid/1,
          valid/2,
+         is_available/1,
+         is_available/2,
 
          verify_otp_vsn/1,
          has_all_artifacts/1,
@@ -66,13 +71,16 @@
 -include("rebar.hrl").
 -include_lib("providers/include/providers.hrl").
 
--export_type([t/0]).
+-export_type([t/0,
+              project_type/0]).
+
+-type project_type() :: rebar3 | mix | undefined.
 
 -record(app_info_t, {name               :: binary() | undefined,
                      app_file_src       :: file:filename_all() | undefined,
                      app_file_src_script:: file:filename_all() | undefined,
                      app_file           :: file:filename_all() | undefined,
-                     original_vsn       :: binary() | string() | undefined,
+                     original_vsn       :: binary() | undefined,
                      parent=root        :: binary() | root,
                      app_details=[]     :: list(),
                      applications=[]    :: list(),
@@ -83,11 +91,13 @@
                      dep_level=0        :: integer(),
                      dir                :: file:name(),
                      out_dir            :: file:name(),
-                     resource_type      :: pkg | src | undefined,
+                     ebin_dir           :: file:name(),
                      source             :: string() | tuple() | checkout | undefined,
                      is_lock=false      :: boolean(),
                      is_checkout=false  :: boolean(),
-                     valid              :: boolean() | undefined}).
+                     valid              :: boolean() | undefined,
+                     project_type       :: project_type(),
+                     is_available=false :: boolean()}).
 
 %%============================================================================
 %% types
@@ -123,7 +133,8 @@ new(AppName, Vsn, Dir) ->
     {ok, #app_info_t{name=rebar_utils:to_binary(AppName),
                      original_vsn=Vsn,
                      dir=rebar_utils:to_list(Dir),
-                     out_dir=rebar_utils:to_list(Dir)}}.
+                     out_dir=rebar_utils:to_list(Dir),
+                     ebin_dir=filename:join(rebar_utils:to_list(Dir), "ebin")}}.
 
 %% @doc build a complete version of the app info with all fields set.
 -spec new(atom() | binary() | string(), binary() | string(), file:name(), list()) ->
@@ -133,6 +144,7 @@ new(AppName, Vsn, Dir, Deps) ->
                      original_vsn=Vsn,
                      dir=rebar_utils:to_list(Dir),
                      out_dir=rebar_utils:to_list(Dir),
+                     ebin_dir=filename:join(rebar_utils:to_list(Dir), "ebin"),
                      deps=Deps}}.
 
 %% @doc build a complete version of the app info with all fields set.
@@ -144,18 +156,21 @@ new(Parent, AppName, Vsn, Dir, Deps) ->
                      original_vsn=Vsn,
                      dir=rebar_utils:to_list(Dir),
                      out_dir=rebar_utils:to_list(Dir),
+                     ebin_dir=filename:join(rebar_utils:to_list(Dir), "ebin"),
                      deps=Deps}}.
 
 %% @doc update the opts based on the contents of a config
 %% file for the app
 -spec update_opts(t(), rebar_dict(), [any()]) -> t().
 update_opts(AppInfo, Opts, Config) ->
-    LockDeps = case resource_type(AppInfo) of
-                   pkg ->
-                       Deps = deps(AppInfo),
-                       [{{locks, default}, Deps}, {{deps, default}, Deps}];
+    LockDeps = case source(AppInfo) of
+                   Tuple when is_tuple(Tuple) andalso element(1, Tuple) =:= pkg ->
+                       %% Deps are set separate for packages
+                       %% instead of making it seem we have no deps
+                       %% don't set anything here.
+                       [];
                    _ ->
-                       deps_from_config(dir(AppInfo), Config)
+                       deps_from_config(dir(AppInfo), proplists:get_value(deps, Config, []))
                end,
 
     Plugins = proplists:get_value(plugins, Config, []),
@@ -165,15 +180,32 @@ update_opts(AppInfo, Opts, Config) ->
 
     NewOpts = rebar_opts:merge_opts(LocalOpts, Opts),
 
-    AppInfo#app_info_t{opts=NewOpts
-                      ,default=NewOpts}.
+    AppInfo#app_info_t{opts=NewOpts,
+                       default=NewOpts}.
+
+%% @doc update current app info opts by merging in a new dict of opts
+-spec update_opts(t(), rebar_dict()) -> t().
+update_opts(AppInfo=#app_info_t{opts=LocalOpts}, Opts) ->
+    NewOpts = rebar_opts:merge_opts(LocalOpts, Opts),
+    AppInfo#app_info_t{opts=NewOpts,
+                       default=NewOpts}.
+
+%% @doc update the opts based on new deps, usually from an app's hex registry metadata
+-spec update_opts_deps(t(), [any()]) -> t().
+update_opts_deps(AppInfo=#app_info_t{opts=Opts}, Deps) ->
+    LocalOpts = dict:from_list([{{locks, default}, Deps}, {{deps, default}, Deps}]),
+    NewOpts = rebar_opts:merge_opts(LocalOpts, Opts),
+    AppInfo#app_info_t{opts=NewOpts,
+                       default=NewOpts,
+                       deps=Deps}.
+
 
 %% @private extract the deps for an app in `Dir' based on its config file data
 -spec deps_from_config(file:filename(), [any()]) -> [{tuple(), any()}, ...].
-deps_from_config(Dir, Config) ->
+deps_from_config(Dir, ConfigDeps) ->
     case rebar_config:consult_lock_file(filename:join(Dir, ?LOCK_FILE)) of
         [] ->
-            [{{deps, default}, proplists:get_value(deps, Config, [])}];
+            [{{deps, default}, ConfigDeps}];
         D ->
             %% We want the top level deps only from the lock file.
             %% This ensures deterministic overrides for configs.
@@ -350,13 +382,13 @@ parent(AppInfo=#app_info_t{}, Parent) ->
 
 %% @doc returns the original version of the app (unevaluated if
 %% asking for a semver)
--spec original_vsn(t()) -> string().
+-spec original_vsn(t()) -> binary().
 original_vsn(#app_info_t{original_vsn=Vsn}) ->
     Vsn.
 
 %% @doc stores the original version of the app (unevaluated if
 %% asking for a semver)
--spec original_vsn(t(), string()) -> t().
+-spec original_vsn(t(), binary() | string()) -> t().
 original_vsn(AppInfo=#app_info_t{}, Vsn) ->
     AppInfo#app_info_t{original_vsn=Vsn}.
 
@@ -426,28 +458,27 @@ out_dir(#app_info_t{out_dir=OutDir}) ->
 %% should go
 -spec out_dir(t(), file:name()) -> t().
 out_dir(AppInfo=#app_info_t{}, OutDir) ->
-    AppInfo#app_info_t{out_dir=rebar_utils:to_list(OutDir)}.
+    AppInfo#app_info_t{out_dir=rebar_utils:to_list(OutDir),
+                       ebin_dir=filename:join(rebar_utils:to_list(OutDir), "ebin")}.
 
 %% @doc gets the directory where ebin files for the app should go
 -spec ebin_dir(t()) -> file:name().
-ebin_dir(#app_info_t{out_dir=OutDir}) ->
-    rebar_utils:to_list(filename:join(OutDir, "ebin")).
+ebin_dir(#app_info_t{ebin_dir=undefined,
+                     out_dir=OutDir}) ->
+    filename:join(rebar_utils:to_list(OutDir), "ebin");
+ebin_dir(#app_info_t{ebin_dir=EbinDir}) ->
+    EbinDir.
+
+%% @doc sets the directory where beam files should go
+-spec ebin_dir(t(), file:name()) -> t().
+ebin_dir(AppInfo, EbinDir) ->
+    AppInfo#app_info_t{ebin_dir=EbinDir}.
 
 %% @doc gets the directory where private files for the app should go
 -spec priv_dir(t()) -> file:name().
 priv_dir(#app_info_t{out_dir=OutDir}) ->
     rebar_utils:to_list(filename:join(OutDir, "priv")).
 
-%% @doc returns whether the app is source app or a package app.
--spec resource_type(t()) -> pkg | src.
-resource_type(#app_info_t{resource_type=ResourceType}) ->
-    ResourceType.
-
-%% @doc sets whether the app is source app or a package app.
--spec resource_type(t(), pkg | src) -> t().
-resource_type(AppInfo=#app_info_t{}, Type) ->
-    AppInfo#app_info_t{resource_type=Type}.
-
 %% @doc finds the source specification for the app
 -spec source(t()) -> string() | tuple().
 source(#app_info_t{source=Source}) ->
@@ -478,6 +509,28 @@ is_checkout(#app_info_t{is_checkout=IsCheckout}) ->
 is_checkout(AppInfo=#app_info_t{}, IsCheckout) ->
     AppInfo#app_info_t{is_checkout=IsCheckout}.
 
+%% @doc returns whether the app source exists in the deps dir
+-spec is_available(t()) -> boolean().
+is_available(#app_info_t{is_available=IsAvailable}) ->
+    IsAvailable.
+
+%% @doc sets whether the app's source is available
+%% only set if the app's source is found in the expected dep directory
+-spec is_available(t(), boolean()) -> t().
+is_available(AppInfo=#app_info_t{}, IsAvailable) ->
+    AppInfo#app_info_t{is_available=IsAvailable}.
+
+%% @doc
+-spec project_type(t()) -> atom().
+project_type(#app_info_t{project_type=ProjectType}) ->
+    ProjectType.
+
+%% @doc
+-spec project_type(t(), atom()) -> t().
+project_type(AppInfo=#app_info_t{}, ProjectType) ->
+    AppInfo#app_info_t{project_type=ProjectType}.
+
+
 %% @doc returns whether the app is valid (built) or not
 -spec valid(t()) -> boolean().
 valid(AppInfo=#app_info_t{valid=undefined}) ->

src/rebar_app_utils.erl

@@ -217,26 +217,23 @@ parse_dep(_, Dep, _, _, _) ->
 dep_to_app(Parent, DepsDir, Name, Vsn, Source, IsLock, State) ->
     CheckoutsDir = rebar_utils:to_list(rebar_dir:checkouts_dir(State, Name)),
     AppInfo = case rebar_app_info:discover(CheckoutsDir) of
-                        {ok, App} ->
-                            rebar_app_info:source(rebar_app_info:is_checkout(App, true), checkout);
-                        not_found ->
-                            Dir = rebar_utils:to_list(filename:join(DepsDir, Name)),
-                            {ok, AppInfo0} =
-                                case rebar_app_info:discover(Dir) of
-                                    {ok, App} ->
-                                        {ok, rebar_app_info:parent(App, Parent)};
-                                    not_found ->
-                                        rebar_app_info:new(Parent, Name, Vsn, Dir, [])
-                                end,
-                                rebar_app_info:source(AppInfo0, Source)
-                    end,
-    C = rebar_config:consult(rebar_app_info:dir(AppInfo)),
-    AppInfo1 = rebar_app_info:update_opts(AppInfo, rebar_app_info:opts(AppInfo), C),
-    Overrides = rebar_state:get(State, overrides, []),
-    AppInfo2 = rebar_app_info:set(AppInfo1, overrides, rebar_app_info:get(AppInfo, overrides, [])++Overrides),
-    AppInfo3 = rebar_app_info:apply_overrides(rebar_app_info:get(AppInfo2, overrides, []), AppInfo2),
-    AppInfo4 = rebar_app_info:apply_profiles(AppInfo3, [default, prod]),
-    AppInfo5 = rebar_app_info:profiles(AppInfo4, [default]),
+                  {ok, App} ->
+                      rebar_app_info:source(rebar_app_info:is_checkout(App, true), checkout);
+                  not_found ->
+                      Dir = rebar_utils:to_list(filename:join(DepsDir, Name)),
+                      {ok, AppInfo0} =
+                          case rebar_app_info:discover(Dir) of
+                              {ok, App} ->
+                                  {ok, rebar_app_info:is_available(rebar_app_info:parent(App, Parent),
+                                                                   true)};
+                              not_found ->
+                                  rebar_app_info:new(Parent, Name, Vsn, Dir, [])
+                          end,
+                      rebar_app_info:source(AppInfo0, Source)
+              end,
+    Overrides = rebar_app_info:get(AppInfo, overrides, []) ++ rebar_state:get(State, overrides, []),
+    AppInfo2 = rebar_app_info:set(AppInfo, overrides, Overrides),
+    AppInfo5 = rebar_app_info:profiles(AppInfo2, [default]),
     rebar_app_info:is_lock(AppInfo5, IsLock).
 
 %% @doc Takes a given application app_info record along with the project.
@@ -250,52 +247,38 @@ expand_deps_sources(Dep, State) ->
 %% around version if required.
 -spec update_source(rebar_app_info:t(), Source, rebar_state:t()) ->
     rebar_app_info:t() when
-      Source :: tuple() | atom() | binary(). % TODO: meta to source()
+      Source :: rebar_resource_v2:source().
 update_source(AppInfo, {pkg, PkgName, PkgVsn, Hash}, State) ->
-    {PkgName1, PkgVsn1} = case PkgVsn of
-                              undefined ->
-                                  get_package(PkgName, "0", State);
-                              <<"~>", Vsn/binary>> ->
-                                  [Vsn1] = [X || X <- binary:split(Vsn, [<<" ">>], [global]), X =/= <<>>],
-                                  get_package(PkgName, Vsn1, State);
-                              _ ->
-                                  {PkgName, PkgVsn}
-                          end,
-    %% store the expected hash for the dependency
-    Hash1 = case Hash of
-        undefined -> % unknown, define the hash since we know the dep
-            fetch_checksum(PkgName1, PkgVsn1, Hash, State);
-        _ -> % keep as is
-            Hash
-    end,
-    AppInfo1 = rebar_app_info:source(AppInfo, {pkg, PkgName1, PkgVsn1, Hash1}),
-    Deps = rebar_packages:deps(PkgName1
-                              ,PkgVsn1
-                              ,State),
-    AppInfo2 = rebar_app_info:resource_type(rebar_app_info:deps(AppInfo1, Deps), pkg),
-    rebar_app_info:original_vsn(AppInfo2, PkgVsn1);
+    case rebar_packages:resolve_version(PkgName, PkgVsn, Hash,
+                                        ?PACKAGE_TABLE, State) of
+        {ok, Package, RepoConfig} ->
+            #package{key={_, PkgVsn1, _},
+                     checksum=Hash1,
+                     dependencies=Deps,
+                     retired=Retired} = Package,
+            maybe_warn_retired(PkgName, PkgVsn1, Hash, Retired),
+            PkgVsn2 = list_to_binary(lists:flatten(ec_semver:format(PkgVsn1))),
+            AppInfo1 = rebar_app_info:source(AppInfo, {pkg, PkgName, PkgVsn2, Hash1, RepoConfig}),
+            AppInfo2 = rebar_app_info:update_opts_deps(AppInfo1, Deps),
+            rebar_app_info:original_vsn(AppInfo2, PkgVsn2);
+        not_found ->
+            throw(?PRV_ERROR({missing_package, PkgName, PkgVsn}));
+        {error, {invalid_vsn, InvalidVsn}} ->
+            throw(?PRV_ERROR({invalid_vsn, PkgName, InvalidVsn}))
+    end;
 update_source(AppInfo, Source, _State) ->
     rebar_app_info:source(AppInfo, Source).
 
-%% @doc grab the checksum for a given package
--spec fetch_checksum(atom(), string(), iodata() | undefined, rebar_state:t()) ->
-    iodata() | no_return().
-fetch_checksum(PkgName, PkgVsn, Hash, State) ->
-    try
-        rebar_packages:registry_checksum({pkg, PkgName, PkgVsn, Hash}, State)
-    catch
-        _:_ ->
-            ?INFO("Package ~ts-~ts not found. Fetching registry updates and trying again...", [PkgName, PkgVsn]),
-            {ok, _} = rebar_prv_update:do(State),
-            rebar_packages:registry_checksum({pkg, PkgName, PkgVsn, Hash}, State)
-    end.
-
 %% @doc convert a given exception's payload into an io description.
 -spec format_error(any()) -> iolist().
-format_error({missing_package, Package}) ->
-    io_lib:format("Package not found in registry: ~ts", [Package]);
+format_error({missing_package, Name, undefined}) ->
+    io_lib:format("Package not found in any repo: ~ts", [rebar_utils:to_binary(Name)]);
+format_error({missing_package, Name, Constraint}) ->
+    io_lib:format("Package not found in any repo: ~ts ~ts", [Name, Constraint]);
 format_error({parse_dep, Dep}) ->
     io_lib:format("Failed parsing dep ~p", [Dep]);
+format_error({invalid_vsn, Dep, InvalidVsn}) ->
+    io_lib:format("Dep ~ts has invalid version ~ts", [Dep, InvalidVsn]);
 format_error(Error) ->
     io_lib:format("~p", [Error]).
 
@@ -303,17 +286,31 @@ format_error(Error) ->
 %% Internal functions
 %% ===================================================================
 
-%% @private find the correct version of a package based on the version
-%% and name passed in.
--spec get_package(binary(), binary() | string(), rebar_state:t()) ->
-    term() | no_return().
-get_package(Dep, Vsn, State) ->
-    case rebar_packages:find_highest_matching(Dep, Vsn, ?PACKAGE_TABLE, State) of
-        {ok, HighestDepVsn} ->
-            {Dep, HighestDepVsn};
-        none ->
-            throw(?PRV_ERROR({missing_package, rebar_utils:to_binary(Dep)}))
-    end.
+maybe_warn_retired(_, _, _, false) ->
+    ok;
+maybe_warn_retired(_, _, Hash, _) when is_binary(Hash) ->
+    %% don't warn if this is a lock
+    ok;
+maybe_warn_retired(Name, Vsn, _, R=#{reason := Reason}) ->
+    Message = maps:get(message, R, ""),
+    ?WARN("Warning: package ~s-~s is retired: (~s) ~s",
+          [Name, ec_semver:format(Vsn), retire_reason(Reason), Message]);
+maybe_warn_retired(_, _, _, _) ->
+    ok.
+
+%% TODO: move to hex_core
+retire_reason('RETIRED_OTHER') ->
+    "other";
+retire_reason('RETIRED_INVALID') ->
+    "invalid";
+retire_reason('RETIRED_SECURITY') ->
+    "security";
+retire_reason('RETIRED_DEPRECATED') ->
+    "deprecated";
+retire_reason('RETIRED_RENAMED') ->
+    "renamed";
+retire_reason(_Other) ->
+    "other".
 
 %% @private checks that all the beam files have been properly
 %% created.

src/rebar_base_compiler.erl

@@ -33,6 +33,8 @@
          run/8,
          ok_tuple/2,
          error_tuple/4,
+         report/1,
+         maybe_report/1,
          format_error_source/2]).
 
 -type desc() :: term().
@@ -94,14 +96,14 @@ run(Config, FirstFiles, SourceDir, SourceExt, TargetDir, TargetExt,
       TargetDir :: file:filename(),
       SourceExt :: string(),
       TargetExt :: string().
-run(Config, FirstFiles, SourceDir, SourceExt, TargetDir, TargetExt,
+run(Config, FirstFiles, SourceDirs, SourceExt, TargetDir, TargetExt,
     Compile3Fn, Opts) ->
     %% Convert simple extension to proper regex
     SourceExtRe = "^(?!\\._).*\\" ++ SourceExt ++ [$$],
 
     Recursive = proplists:get_value(recursive, Opts, true),
     %% Find all possible source files
-    FoundFiles = rebar_utils:find_files(SourceDir, SourceExtRe, Recursive),
+    FoundFiles = rebar_utils:find_files_in_dirs(SourceDirs, SourceExtRe, Recursive),
     %% Remove first files from found files
     RestFiles = [Source || Source <- FoundFiles,
                            not lists:member(Source, FirstFiles)],
@@ -111,7 +113,7 @@ run(Config, FirstFiles, SourceDir, SourceExt, TargetDir, TargetExt,
 
     run(Config, FirstFiles, RestFiles,
         fun(S, C) ->
-                Target = target_file(S, SourceDir, SourceExt,
+                Target = target_file(S, SourceExt,
                                      TargetDir, TargetExt),
                 simple_compile_wrapper(S, Target, Compile3Fn, C, CheckLastMod)
         end).
@@ -160,32 +162,15 @@ simple_compile_wrapper(Source, Target, Compile3Fn, Config, true) ->
 
 %% @private take a basic source set of file fragments and a target location,
 %% create a file path and name for a compile artifact.
--spec target_file(SourceFile, SourceDir, SourceExt, TargetDir, TargetExt) -> File when
+-spec target_file(SourceFile, SourceExt, TargetDir, TargetExt) -> File when
       SourceFile :: file:filename(),
-      SourceDir :: file:filename(),
       TargetDir :: file:filename(),
       SourceExt :: string(),
       TargetExt :: string(),
       File :: file:filename().
-target_file(SourceFile, SourceDir, SourceExt, TargetDir, TargetExt) ->
-    BaseFile = remove_common_path(SourceFile, SourceDir),
-    filename:join([TargetDir, filename:basename(BaseFile, SourceExt) ++ TargetExt]).
-
-%% @private removes the common prefix between two file paths.
-%% The remainder of the first file path passed will have its ending returned
-%% when either path starts diverging.
--spec remove_common_path(file:filename(), file:filename()) -> file:filename().
-remove_common_path(Fname, Path) ->
-    remove_common_path1(filename:split(Fname), filename:split(Path)).
-
-%% @private given two lists of file fragments, discard the identical
-%% prefixed sections, and return the final bit of the first operand
-%% as a filename.
--spec remove_common_path1([string()], [string()]) -> file:filename().
-remove_common_path1([Part | RestFilename], [Part | RestPath]) ->
-    remove_common_path1(RestFilename, RestPath);
-remove_common_path1(FilenameParts, _) ->
-    filename:join(FilenameParts).
+target_file(SourceFile, SourceExt, TargetDir, TargetExt) ->
+    %% BaseFile = remove_common_path(SourceFile, SourceDir),
+    filename:join([TargetDir, filename:basename(SourceFile, SourceExt) ++ TargetExt]).
 
 %% @private runs the compile function `CompileFn' on every file
 %% passed internally, along with the related project configuration.
@@ -264,14 +249,24 @@ report(Messages) ->
 -spec format_errors(_, Extra, [err_or_warn()]) -> [string()] when
       Extra :: string().
 format_errors(_MainSource, Extra, Errors) ->
-    [begin
-         [format_error(Source, Extra, Desc) || Desc <- Descs]
-     end
+    [[format_error(Source, Extra, Desc) || Desc <- Descs]
      || {Source, Descs} <- Errors].
 
 %% @private format compiler errors into proper outputtable strings
 -spec format_error(file:filename(), Extra, err_or_warn()) -> string() when
       Extra :: string().
+format_error(Source, Extra, {Line, Mod=epp, Desc={include,lib,File}}) ->
+    %% Special case for include file errors, overtaking the default one
+    BaseDesc = Mod:format_error(Desc),
+    Friendly = case filename:split(File) of
+        [Lib, "include", _] ->
+            io_lib:format("; Make sure ~s is in your app "
+                          "file's 'applications' list", [Lib]);
+        _ ->
+            ""
+    end,
+    FriendlyDesc = BaseDesc ++ Friendly,
+    ?FMT("~ts:~w: ~ts~ts~n", [Source, Line, Extra, FriendlyDesc]);
 format_error(Source, Extra, {{Line, Column}, Mod, Desc}) ->
     ErrorDesc = Mod:format_error(Desc),
     ?FMT("~ts:~w:~w: ~ts~ts~n", [Source, Line, Column, Extra, ErrorDesc]);

src/rebar_compiler.erl

@@ -0,0 +1,303 @@
+-module(rebar_compiler).
+
+-export([compile_all/2,
+         clean/2,
+
+         ok_tuple/2,
+         error_tuple/4,
+         maybe_report/1,
+         format_error_source/2,
+         report/1]).
+
+-include("rebar.hrl").
+
+-type extension() :: string().
+-type out_mappings() :: [{extension(), file:filename()}].
+
+-callback context(rebar_app_info:t()) -> #{src_dirs     => [file:dirname()],
+                                           include_dirs => [file:dirname()],
+                                           src_ext      => extension(),
+                                           out_mappings => out_mappings()}.
+-callback needed_files(digraph:graph(), [file:filename()], rebar_app_info:t()) -> [file:filename()].
+-callback dependencies(file:filename(), file:dirname(), [file:dirname()]) -> [file:filename()].
+-callback compile(file:filename(), out_mappings(), rebar_dict(), list()) ->
+    ok | {ok, [string()]} | {ok, [string()], [string()]}.
+
+-define(DAG_VSN, 2).
+-define(DAG_FILE, "source.dag").
+-type dag_v() :: {digraph:vertex(), term()} | 'false'.
+-type dag_e() :: {digraph:vertex(), digraph:vertex()}.
+-type dag() :: {list(dag_v()), list(dag_e()), list(string())}.
+-record(dag, {vsn = ?DAG_VSN :: pos_integer(),
+              info = {[], [], []} :: dag()}).
+
+-define(RE_PREFIX, "^(?!\\._)").
+
+compile_all(Compilers, AppInfo) ->
+    EbinDir = rebar_utils:to_list(rebar_app_info:ebin_dir(AppInfo)),
+    %% Make sure that outdir is on the path
+    ok = rebar_file_utils:ensure_dir(EbinDir),
+    true = code:add_patha(filename:absname(EbinDir)),
+
+    %% necessary for erlang:function_exported/3 to work as expected
+    %% called here for clarity as it's required by both opts_changed/2
+    %% and erl_compiler_opts_set/0 in needed_files
+    _ = code:ensure_loaded(compile),
+
+    lists:foreach(fun(CompilerMod) ->
+                          run(CompilerMod, AppInfo),
+                          run_on_extra_src_dirs(CompilerMod, AppInfo, fun run/2)
+                  end, Compilers),
+    ok.
+
+run(CompilerMod, AppInfo) ->
+    #{src_dirs := SrcDirs,
+      include_dirs := InclDirs,
+      src_ext := SrcExt,
+      out_mappings := Mappings} = CompilerMod:context(AppInfo),
+
+    BaseDir = rebar_utils:to_list(rebar_app_info:dir(AppInfo)),
+    EbinDir = rebar_utils:to_list(rebar_app_info:ebin_dir(AppInfo)),
+
+    BaseOpts = rebar_app_info:opts(AppInfo),
+    AbsInclDirs = [filename:join(BaseDir, InclDir) || InclDir <- InclDirs],
+    FoundFiles = find_source_files(BaseDir, SrcExt, SrcDirs, BaseOpts),
+
+    OutDir = rebar_app_info:out_dir(AppInfo),
+    AbsSrcDirs = [filename:join(BaseDir, SrcDir) || SrcDir <- SrcDirs],
+    G = init_dag(CompilerMod, AbsInclDirs, AbsSrcDirs, FoundFiles, OutDir, EbinDir),
+    {{FirstFiles, FirstFileOpts}, {RestFiles, Opts}} = CompilerMod:needed_files(G, FoundFiles, AppInfo),
+    true = digraph:delete(G),
+
+    compile_each(FirstFiles, FirstFileOpts, BaseOpts, Mappings, CompilerMod),
+    compile_each(RestFiles, Opts, BaseOpts, Mappings, CompilerMod).
+
+compile_each([], _Opts, _Config, _Outs, _CompilerMod) ->
+    ok;
+compile_each([Source | Rest], Opts, Config, Outs, CompilerMod) ->
+    case CompilerMod:compile(Source, Outs, Config, Opts) of
+        ok ->
+            ?DEBUG("~tsCompiled ~ts", [rebar_utils:indent(1), filename:basename(Source)]);
+        {ok, Warnings} ->
+            report(Warnings),
+            ?DEBUG("~tsCompiled ~ts", [rebar_utils:indent(1), filename:basename(Source)]);
+        skipped ->
+            ?DEBUG("~tsSkipped ~ts", [rebar_utils:indent(1), filename:basename(Source)]);
+        Error ->
+            NewSource = format_error_source(Source, Config),
+            ?ERROR("Compiling ~ts failed", [NewSource]),
+            maybe_report(Error),
+            ?DEBUG("Compilation failed: ~p", [Error]),
+            ?FAIL
+    end,
+    compile_each(Rest, Opts, Config, Outs, CompilerMod).
+
+%% @doc remove compiled artifacts from an AppDir.
+-spec clean([module()], rebar_app_info:t()) -> 'ok'.
+clean(Compilers, AppInfo) ->
+    lists:foreach(fun(CompilerMod) ->
+                          clean_(CompilerMod, AppInfo),
+                          run_on_extra_src_dirs(CompilerMod, AppInfo, fun clean_/2)
+                  end, Compilers).
+
+clean_(CompilerMod, AppInfo) ->
+    #{src_dirs := SrcDirs,
+      src_ext := SrcExt} = CompilerMod:context(AppInfo),
+    BaseDir = rebar_app_info:dir(AppInfo),
+    Opts = rebar_app_info:opts(AppInfo),
+    EbinDir = rebar_app_info:ebin_dir(AppInfo),
+
+    FoundFiles = find_source_files(BaseDir, SrcExt, SrcDirs, Opts),
+    CompilerMod:clean(FoundFiles, AppInfo),
+    rebar_file_utils:rm_rf(dag_file(CompilerMod, EbinDir)).
+
+
+run_on_extra_src_dirs(CompilerMod, AppInfo, Fun) ->
+    ExtraDirs = rebar_dir:extra_src_dirs(rebar_app_info:opts(AppInfo), []),
+    run_on_extra_src_dirs(ExtraDirs, CompilerMod, AppInfo, Fun).
+
+run_on_extra_src_dirs([], _CompilerMod, _AppInfo, _Fun) ->
+    ok;
+run_on_extra_src_dirs([Dir | Rest], CompilerMod, AppInfo, Fun) ->
+    case filelib:is_dir(filename:join(rebar_app_info:dir(AppInfo), Dir)) of
+        true ->
+            EbinDir = filename:join(rebar_app_info:out_dir(AppInfo), Dir),
+            AppInfo1 = rebar_app_info:ebin_dir(AppInfo, EbinDir),
+            AppInfo2 = rebar_app_info:set(AppInfo1, src_dirs, [Dir]),
+            AppInfo3 = rebar_app_info:set(AppInfo2, extra_src_dirs, ["src"]),
+            Fun(CompilerMod, AppInfo3);
+        _ ->
+            ok
+    end,
+    run_on_extra_src_dirs(Rest, CompilerMod, AppInfo, Fun).
+
+%% These functions are here for the ultimate goal of getting rid of
+%% rebar_base_compiler. This can't be done because of existing plugins.
+
+ok_tuple(Source, Ws) ->
+    rebar_base_compiler:ok_tuple(Source, Ws).
+
+error_tuple(Source, Es, Ws, Opts) ->
+    rebar_base_compiler:error_tuple(Source, Es, Ws, Opts).
+
+maybe_report(Reportable) ->
+    rebar_base_compiler:maybe_report(Reportable).
+
+format_error_source(Path, Opts) ->
+    rebar_base_compiler:format_error_source(Path, Opts).
+
+report(Messages) ->
+    rebar_base_compiler:report(Messages).
+
+%% private functions
+
+find_source_files(BaseDir, SrcExt, SrcDirs, Opts) ->
+    SourceExtRe = "^(?!\\._).*\\" ++ SrcExt ++ [$$],
+    lists:flatmap(fun(SrcDir) ->
+                      Recursive = rebar_dir:recursive(Opts, SrcDir),
+                      rebar_utils:find_files_in_dirs([filename:join(BaseDir, SrcDir)], SourceExtRe, Recursive)
+                  end, SrcDirs).
+
+dag_file(CompilerMod, Dir) ->
+    filename:join([rebar_dir:local_cache_dir(Dir), CompilerMod, ?DAG_FILE]).
+
+%% private graph functions
+
+%% Get dependency graph of given Erls files and their dependencies (header files,
+%% parse transforms, behaviours etc.) located in their directories or given
+%% InclDirs. Note that last modification times stored in vertices already respect
+%% dependencies induced by given graph G.
+init_dag(Compiler, InclDirs, SrcDirs, Erls, Dir, EbinDir) ->
+    G = digraph:new([acyclic]),
+    try restore_dag(Compiler, G, InclDirs, Dir)
+    catch
+        _:_ ->
+            ?WARN("Failed to restore ~ts file. Discarding it.~n", [dag_file(Compiler, Dir)]),
+            file:delete(dag_file(Compiler, Dir))
+    end,
+    Dirs = lists:usort(InclDirs ++ SrcDirs),
+    %% A source file may have been renamed or deleted. Remove it from the graph
+    %% and remove any beam file for that source if it exists.
+    Modified = maybe_rm_beams_and_edges(G, EbinDir, Erls),
+    Modified1 = lists:foldl(update_dag_fun(G, Compiler, Dirs), Modified, Erls),
+    if Modified1 -> store_dag(Compiler, G, InclDirs, Dir); not Modified1 -> ok end,
+    G.
+
+maybe_rm_beams_and_edges(G, Dir, Files) ->
+    Vertices = digraph:vertices(G),
+    case lists:filter(fun(File) ->
+                              case filename:extension(File) =:= ".erl" of
+                                  true ->
+                                      maybe_rm_beam_and_edge(G, Dir, File);
+                                  false ->
+                                      false
+                              end
+                      end, lists:sort(Vertices) -- lists:sort(Files)) of
+        [] ->
+            false;
+        _ ->
+            true
+    end.
+
+maybe_rm_beam_and_edge(G, OutDir, Source) ->
+    %% This is NOT a double check it is the only check that the source file is actually gone
+    case filelib:is_regular(Source) of
+        true ->
+            %% Actually exists, don't delete
+            false;
+        false ->
+            Target = target_base(OutDir, Source) ++ ".beam",
+            ?DEBUG("Source ~ts is gone, deleting previous beam file if it exists ~ts", [Source, Target]),
+            file:delete(Target),
+            digraph:del_vertex(G, Source),
+            true
+    end.
+
+
+target_base(OutDir, Source) ->
+    filename:join(OutDir, filename:basename(Source, ".erl")).
+
+restore_dag(Compiler, G, InclDirs, Dir) ->
+    case file:read_file(dag_file(Compiler, Dir)) of
+        {ok, Data} ->
+            % Since externally passed InclDirs can influence dependency graph (see
+            % modify_dag), we have to check here that they didn't change.
+            #dag{vsn=?DAG_VSN, info={Vs, Es, InclDirs}} =
+                binary_to_term(Data),
+            lists:foreach(
+              fun({V, LastUpdated}) ->
+                      digraph:add_vertex(G, V, LastUpdated)
+              end, Vs),
+            lists:foreach(
+              fun({_, V1, V2, _}) ->
+                      digraph:add_edge(G, V1, V2)
+              end, Es);
+        {error, _} ->
+            ok
+    end.
+
+store_dag(Compiler, G, InclDirs, Dir) ->
+    Vs = lists:map(fun(V) -> digraph:vertex(G, V) end, digraph:vertices(G)),
+    Es = lists:map(fun(E) -> digraph:edge(G, E) end, digraph:edges(G)),
+    File = dag_file(Compiler, Dir),
+    ok = filelib:ensure_dir(File),
+    Data = term_to_binary(#dag{info={Vs, Es, InclDirs}}, [{compressed, 2}]),
+    file:write_file(File, Data).
+
+update_dag(G, Compiler, Dirs, Source) ->
+    case digraph:vertex(G, Source) of
+        {_, LastUpdated} ->
+            case filelib:last_modified(Source) of
+                0 ->
+                    %% The file doesn't exist anymore,
+                    %% erase it from the graph.
+                    %% All the edges will be erased automatically.
+                    digraph:del_vertex(G, Source),
+                    modified;
+                LastModified when LastUpdated < LastModified ->
+                    modify_dag(G, Compiler, Source, LastModified, filename:dirname(Source), Dirs);
+                _ ->
+                    Modified = lists:foldl(
+                        update_dag_fun(G, Compiler, Dirs),
+                        false, digraph:out_neighbours(G, Source)),
+                    MaxModified = update_max_modified_deps(G, Source),
+                    case Modified orelse MaxModified > LastUpdated of
+                        true -> modified;
+                        false -> unmodified
+                    end
+            end;
+        false ->
+            modify_dag(G, Compiler, Source, filelib:last_modified(Source), filename:dirname(Source), Dirs)
+    end.
+
+modify_dag(G, Compiler, Source, LastModified, SourceDir, Dirs) ->
+    AbsIncls = Compiler:dependencies(Source, SourceDir, Dirs),
+    digraph:add_vertex(G, Source, LastModified),
+    digraph:del_edges(G, digraph:out_edges(G, Source)),
+    lists:foreach(
+      fun(Incl) ->
+              update_dag(G, Compiler, Dirs, Incl),
+              digraph:add_edge(G, Source, Incl)
+      end, AbsIncls),
+    modified.
+
+update_dag_fun(G, Compiler, Dirs) ->
+    fun(Erl, Modified) ->
+        case update_dag(G, Compiler, Dirs, Erl) of
+            modified -> true;
+            unmodified -> Modified
+        end
+    end.
+
+update_max_modified_deps(G, Source) ->
+    MaxModified =
+        lists:foldl(fun(File, Acc) ->
+                            case digraph:vertex(G, File) of
+                                {_, MaxModified} when MaxModified > Acc ->
+                                    MaxModified;
+                                _ ->
+                                    Acc
+                            end
+                    end, 0, [Source | digraph:out_neighbours(G, Source)]),
+    digraph:add_vertex(G, Source, MaxModified),
+    MaxModified.

src/rebar_compiler_erl.erl

@@ -0,0 +1,368 @@
+-module(rebar_compiler_erl).
+
+-behaviour(rebar_compiler).
+
+-export([context/1,
+         needed_files/3,
+         dependencies/3,
+         compile/4,
+         clean/2]).
+
+-include("rebar.hrl").
+
+context(AppInfo) ->
+    EbinDir = rebar_app_info:ebin_dir(AppInfo),
+    Mappings = [{".beam", EbinDir}],
+
+    OutDir = rebar_app_info:dir(AppInfo),
+    SrcDirs = rebar_dir:src_dirs(rebar_app_info:opts(AppInfo), ["src"]),
+    ExistingSrcDirs = lists:filter(fun(D) ->
+                                           ec_file:is_dir(filename:join(OutDir, D))
+                                   end, SrcDirs),
+
+    RebarOpts = rebar_app_info:opts(AppInfo),
+    ErlOpts = rebar_opts:erl_opts(RebarOpts),
+    ErlOptIncludes = proplists:get_all_values(i, ErlOpts),
+    InclDirs = lists:map(fun(Incl) -> filename:absname(Incl) end, ErlOptIncludes),
+
+    #{src_dirs => ExistingSrcDirs,
+      include_dirs => [filename:join([OutDir, "include"]) | InclDirs],
+      src_ext => ".erl",
+      out_mappings => Mappings}.
+
+
+needed_files(Graph, FoundFiles, AppInfo) ->
+    OutDir = rebar_app_info:out_dir(AppInfo),
+    Dir = rebar_app_info:dir(AppInfo),
+    EbinDir = rebar_app_info:ebin_dir(AppInfo),
+    RebarOpts = rebar_app_info:opts(AppInfo),
+    ErlOpts = rebar_opts:erl_opts(RebarOpts),
+    ?DEBUG("erlopts ~p", [ErlOpts]),
+    ?DEBUG("files to compile ~p", [FoundFiles]),
+
+    %% Make sure that the ebin dir is on the path
+    ok = rebar_file_utils:ensure_dir(EbinDir),
+    true = code:add_patha(filename:absname(EbinDir)),
+
+    {ParseTransforms, Rest} = split_source_files(FoundFiles, ErlOpts),
+    NeededErlFiles = case needed_files(Graph, ErlOpts, RebarOpts, OutDir, EbinDir, ParseTransforms) of
+                         [] ->
+                             needed_files(Graph, ErlOpts, RebarOpts, OutDir, EbinDir, Rest);
+                         _  ->
+                             %% at least one parse transform in the opts needs updating, so recompile all
+                             FoundFiles
+                     end,
+    {ErlFirstFiles, ErlOptsFirst} = erl_first_files(RebarOpts, ErlOpts, Dir, NeededErlFiles),
+    SubGraph = digraph_utils:subgraph(Graph, NeededErlFiles),
+    DepErlsOrdered = digraph_utils:topsort(SubGraph),
+    OtherErls = lists:reverse(DepErlsOrdered),
+
+    PrivIncludes = [{i, filename:join(OutDir, Src)}
+                    || Src <- rebar_dir:all_src_dirs(RebarOpts, ["src"], [])],
+    AdditionalOpts = PrivIncludes ++ [{i, filename:join(OutDir, "include")}, {i, OutDir}, return],
+
+    true = digraph:delete(SubGraph),
+
+    {{ErlFirstFiles, ErlOptsFirst ++ AdditionalOpts},
+     {[Erl || Erl <- OtherErls,
+              not lists:member(Erl, ErlFirstFiles)], ErlOpts ++ AdditionalOpts}}.
+
+dependencies(Source, SourceDir, Dirs) ->
+    {ok, Fd} = file:open(Source, [read]),
+    Incls = parse_attrs(Fd, [], SourceDir),
+    AbsIncls = expand_file_names(Incls, Dirs),
+    ok = file:close(Fd),
+    AbsIncls.
+
+compile(Source, [{_, OutDir}], Config, ErlOpts) ->
+    case compile:file(Source, [{outdir, OutDir} | ErlOpts]) of
+        {ok, _Mod} ->
+            ok;
+        {ok, _Mod, []} ->
+            ok;
+        {ok, _Mod, Ws} ->
+            FormattedWs = format_error_sources(Ws, Config),
+            rebar_compiler:ok_tuple(Source, FormattedWs);
+        {error, Es, Ws} ->
+            error_tuple(Source, Es, Ws, Config, ErlOpts);
+        error ->
+            error
+    end.
+
+clean(Files, AppInfo) ->
+    EbinDir = rebar_app_info:ebin_dir(AppInfo),
+    [begin
+         Source = filename:basename(File, ".erl"),
+         Target = target_base(EbinDir, Source) ++ ".beam",
+         file:delete(Target)
+     end || File <- Files].
+
+%%
+
+error_tuple(Module, Es, Ws, AllOpts, Opts) ->
+    FormattedEs = format_error_sources(Es, AllOpts),
+    FormattedWs = format_error_sources(Ws, AllOpts),
+    rebar_compiler:error_tuple(Module, FormattedEs, FormattedWs, Opts).
+
+format_error_sources(Es, Opts) ->
+    [{rebar_compiler:format_error_source(Src, Opts), Desc}
+     || {Src, Desc} <- Es].
+
+%% Get files which need to be compiled first, i.e. those specified in erl_first_files
+%% and parse_transform options.  Also produce specific erl_opts for these first
+%% files, so that yet to be compiled parse transformations are excluded from it.
+erl_first_files(Opts, ErlOpts, Dir, NeededErlFiles) ->
+    ErlFirstFilesConf = rebar_opts:get(Opts, erl_first_files, []),
+    valid_erl_first_conf(ErlFirstFilesConf),
+    NeededSrcDirs = lists:usort(lists:map(fun filename:dirname/1, NeededErlFiles)),
+    %% NOTE: order of files here is important!
+    ErlFirstFiles =
+        [filename:join(Dir, File) || File <- ErlFirstFilesConf,
+                                     lists:member(filename:join(Dir, File), NeededErlFiles)],
+    {ParseTransforms, ParseTransformsErls} =
+        lists:unzip(lists:flatmap(
+                      fun(PT) ->
+                              PTerls = [filename:join(D, module_to_erl(PT)) || D <- NeededSrcDirs],
+                              [{PT, PTerl} || PTerl <- PTerls, lists:member(PTerl, NeededErlFiles)]
+                      end, proplists:get_all_values(parse_transform, ErlOpts))),
+    ErlOptsFirst = lists:filter(fun({parse_transform, PT}) ->
+                                        not lists:member(PT, ParseTransforms);
+                                   (_) ->
+                                        true
+                                end, ErlOpts),
+    {ErlFirstFiles ++ ParseTransformsErls, ErlOptsFirst}.
+
+split_source_files(SourceFiles, ErlOpts) ->
+    ParseTransforms = proplists:get_all_values(parse_transform, ErlOpts),
+    lists:partition(fun(Source) ->
+                            lists:member(filename_to_atom(Source), ParseTransforms)
+                    end, SourceFiles).
+
+filename_to_atom(F) -> list_to_atom(filename:rootname(filename:basename(F))).
+
+%% Get subset of SourceFiles which need to be recompiled, respecting
+%% dependencies induced by given graph G.
+needed_files(Graph, ErlOpts, RebarOpts, Dir, OutDir, SourceFiles) ->
+    lists:filter(fun(Source) ->
+                         TargetBase = target_base(OutDir, Source),
+                         Target = TargetBase ++ ".beam",
+                         PrivIncludes = [{i, filename:join(Dir, Src)}
+                                         || Src <- rebar_dir:all_src_dirs(RebarOpts, ["src"], [])],
+                         AllOpts = [{outdir, filename:dirname(Target)}
+                                   ,{i, filename:join(Dir, "include")}
+                                   ,{i, Dir}] ++ PrivIncludes ++ ErlOpts,
+                         digraph:vertex(Graph, Source) > {Source, filelib:last_modified(Target)}
+                              orelse opts_changed(AllOpts, TargetBase)
+                              orelse erl_compiler_opts_set()
+                 end, SourceFiles).
+
+target_base(OutDir, Source) ->
+    filename:join(OutDir, filename:basename(Source, ".erl")).
+
+opts_changed(NewOpts, Target) ->
+    TotalOpts = case erlang:function_exported(compile, env_compiler_options, 0) of
+        true  -> NewOpts ++ compile:env_compiler_options();
+        false -> NewOpts
+    end,
+    case compile_info(Target) of
+        {ok, Opts} -> lists:any(fun effects_code_generation/1, lists:usort(TotalOpts) -- lists:usort(Opts));
+        _          -> true
+    end.
+
+effects_code_generation(Option) ->
+    case Option of
+        beam -> false;
+        report_warnings -> false;
+        report_errors -> false;
+        return_errors-> false;
+        return_warnings-> false;
+        report -> false;
+        warnings_as_errors -> false;
+        binary -> false;
+        verbose -> false;
+        {cwd,_} -> false;
+        {outdir, _} -> false;
+        _ -> true
+    end.
+
+compile_info(Target) ->
+    case beam_lib:chunks(Target, [compile_info]) of
+        {ok, {_mod, Chunks}} ->
+            CompileInfo = proplists:get_value(compile_info, Chunks, []),
+            {ok, proplists:get_value(options, CompileInfo, [])};
+        {error, beam_lib, Reason} ->
+            ?WARN("Couldn't read debug info from ~p for reason: ~p", [Target, Reason]),
+            {error, Reason}
+    end.
+
+erl_compiler_opts_set() ->
+    EnvSet = case os:getenv("ERL_COMPILER_OPTIONS") of
+        false -> false;
+        _     -> true
+    end,
+    %% return false if changed env opts would have been caught in opts_changed/2
+    EnvSet andalso not erlang:function_exported(compile, env_compiler_options, 0).
+
+valid_erl_first_conf(FileList) ->
+    Strs = filter_file_list(FileList),
+    case rebar_utils:is_list_of_strings(Strs) of
+        true -> true;
+        false -> ?ABORT("An invalid file list (~p) was provided as part of your erl_first_files directive",
+                        [FileList])
+    end.
+
+filter_file_list(FileList) ->
+    Atoms = lists:filter( fun(X) -> is_atom(X) end, FileList),
+    case Atoms of
+        [] ->
+            FileList;
+        _ ->
+          atoms_in_erl_first_files_warning(Atoms),
+          lists:filter( fun(X) -> not(is_atom(X)) end, FileList)
+     end.
+
+atoms_in_erl_first_files_warning(Atoms) ->
+  W = "You have provided atoms as file entries in erl_first_files; "
+      "erl_first_files only expects lists of filenames as strings. "
+      "The following modules (~p) may not work as expected and it is advised "
+      "that you change these entires to string format "
+      "(e.g., \"src/module.erl\") ",
+  ?WARN(W, [Atoms]).
+
+module_to_erl(Mod) ->
+    atom_to_list(Mod) ++ ".erl".
+
+parse_attrs(Fd, Includes, Dir) ->
+    case io:parse_erl_form(Fd, "") of
+        {ok, Form, _Line} ->
+            case erl_syntax:type(Form) of
+                attribute ->
+                    NewIncludes = process_attr(Form, Includes, Dir),
+                    parse_attrs(Fd, NewIncludes, Dir);
+                _ ->
+                    parse_attrs(Fd, Includes, Dir)
+            end;
+        {eof, _} ->
+            Includes;
+        _Err ->
+            parse_attrs(Fd, Includes, Dir)
+    end.
+
+process_attr(Form, Includes, Dir) ->
+    AttrName = erl_syntax:atom_value(erl_syntax:attribute_name(Form)),
+    process_attr(AttrName, Form, Includes, Dir).
+
+process_attr(import, Form, Includes, _Dir) ->
+    case erl_syntax_lib:analyze_import_attribute(Form) of
+        {Mod, _Funs} ->
+            [module_to_erl(Mod)|Includes];
+        Mod ->
+            [module_to_erl(Mod)|Includes]
+    end;
+process_attr(file, Form, Includes, _Dir) ->
+    {File, _} = erl_syntax_lib:analyze_file_attribute(Form),
+    [File|Includes];
+process_attr(include, Form, Includes, _Dir) ->
+    [FileNode] = erl_syntax:attribute_arguments(Form),
+    File = erl_syntax:string_value(FileNode),
+    [File|Includes];
+process_attr(include_lib, Form, Includes, Dir) ->
+    [FileNode] = erl_syntax:attribute_arguments(Form),
+    RawFile = erl_syntax:string_value(FileNode),
+    maybe_expand_include_lib_path(RawFile, Dir) ++ Includes;
+process_attr(behavior, Form, Includes, _Dir) ->
+    process_attr(behaviour, Form, Includes, _Dir);
+process_attr(behaviour, Form, Includes, _Dir) ->
+    [FileNode] = erl_syntax:attribute_arguments(Form),
+    File = module_to_erl(erl_syntax:atom_value(FileNode)),
+    [File|Includes];
+process_attr(compile, Form, Includes, _Dir) ->
+    [Arg] = erl_syntax:attribute_arguments(Form),
+    case erl_syntax:concrete(Arg) of
+        {parse_transform, Mod} ->
+            [module_to_erl(Mod)|Includes];
+        {core_transform, Mod} ->
+            [module_to_erl(Mod)|Includes];
+        L when is_list(L) ->
+            lists:foldl(
+              fun({parse_transform, Mod}, Acc) ->
+                      [module_to_erl(Mod)|Acc];
+                 ({core_transform, Mod}, Acc) ->
+                      [module_to_erl(Mod)|Acc];
+                 (_, Acc) ->
+                      Acc
+              end, Includes, L);
+        _ ->
+            Includes
+    end;
+process_attr(_, _Form, Includes, _Dir) ->
+    Includes.
+
+%% NOTE: If, for example, one of the entries in Files, refers to
+%% gen_server.erl, that entry will be dropped. It is dropped because
+%% such an entry usually refers to the beam file, and we don't pass a
+%% list of OTP src dirs for finding gen_server.erl's full path. Also,
+%% if gen_server.erl was modified, it's not rebar's task to compile a
+%% new version of the beam file. Therefore, it's reasonable to drop
+%% such entries. Also see process_attr(behaviour, Form, Includes).
+-spec expand_file_names([file:filename()],
+                        [file:filename()]) -> [file:filename()].
+expand_file_names(Files, Dirs) ->
+    %% We check if Files exist by itself or within the directories
+    %% listed in Dirs.
+    %% Return the list of files matched.
+    lists:flatmap(
+      fun(Incl) ->
+              case filelib:is_regular(Incl) of
+                  true ->
+                      [Incl];
+                  false ->
+                      lists:flatmap(
+                        fun(Dir) ->
+                                FullPath = filename:join(Dir, Incl),
+                                case filelib:is_regular(FullPath) of
+                                    true ->
+                                        [FullPath];
+                                    false ->
+                                        []
+                                end
+                        end, Dirs)
+              end
+      end, Files).
+
+%% Given a path like "stdlib/include/erl_compile.hrl", return
+%% "OTP_INSTALL_DIR/lib/erlang/lib/stdlib-x.y.z/include/erl_compile.hrl".
+%% Usually a simple [Lib, SubDir, File1] = filename:split(File) should
+%% work, but to not crash when an unusual include_lib path is used,
+%% utilize more elaborate logic.
+maybe_expand_include_lib_path(File, Dir) ->
+    File1 = filename:basename(File),
+    case filename:split(filename:dirname(File)) of
+        [_] ->
+            warn_and_find_path(File, Dir);
+        [Lib | SubDir] ->
+            case code:lib_dir(list_to_atom(Lib), list_to_atom(filename:join(SubDir))) of
+                {error, bad_name} ->
+                    warn_and_find_path(File, Dir);
+                AppDir ->
+                    [filename:join(AppDir, File1)]
+            end
+    end.
+
+%% The use of -include_lib was probably incorrect by the user but lets try to make it work.
+%% We search in the outdir and outdir/../include to see if the header exists.
+warn_and_find_path(File, Dir) ->
+    SrcHeader = filename:join(Dir, File),
+    case filelib:is_regular(SrcHeader) of
+        true ->
+            [SrcHeader];
+        false ->
+            IncludeDir = filename:join(rebar_utils:droplast(filename:split(Dir))++["include"]),
+            IncludeHeader = filename:join(IncludeDir, File),
+            case filelib:is_regular(IncludeHeader) of
+                true ->
+                    [filename:join(IncludeDir, File)];
+                false ->
+                    []
+            end
+    end.

src/rebar_compiler_mib.erl

@@ -0,0 +1,70 @@
+-module(rebar_compiler_mib).
+
+-behaviour(rebar_compiler).
+
+-export([context/1,
+         needed_files/3,
+         dependencies/3,
+         compile/4,
+         clean/2]).
+
+-include("rebar.hrl").
+-include_lib("stdlib/include/erl_compile.hrl").
+
+context(AppInfo) ->
+    Dir = rebar_app_info:dir(AppInfo),
+    Mappings = [{".bin", filename:join([Dir, "priv", "mibs"])},
+               {".hrl", filename:join(Dir, "include")}],
+
+    #{src_dirs => ["mibs"],
+      include_dirs => [],
+      src_ext => ".mib",
+      out_mappings => Mappings}.
+
+needed_files(_, FoundFiles, AppInfo) ->
+    FirstFiles = [],
+
+    %% Remove first files from found files
+    RestFiles = [Source || Source <- FoundFiles, not lists:member(Source, FirstFiles)],
+
+    Opts = rebar_opts:get(rebar_app_info:opts(AppInfo), mib_opts, []),
+    {{FirstFiles, Opts}, {RestFiles, Opts}}.
+
+dependencies(_, _, _) ->
+    [].
+
+compile(Source, OutDirs, _, Opts) ->
+    {_, BinOut} = lists:keyfind(".bin", 1, OutDirs),
+    {_, HrlOut} = lists:keyfind(".hrl", 1, OutDirs),
+
+    ok = rebar_file_utils:ensure_dir(BinOut),
+    ok = rebar_file_utils:ensure_dir(HrlOut),
+    Mib = filename:join(BinOut, filename:basename(Source, ".mib")),
+    HrlFilename = Mib ++ ".hrl",
+
+    AllOpts = [{outdir, BinOut}, {i, [BinOut]}] ++ Opts,
+
+    case snmpc:compile(Source, AllOpts) of
+        {ok, _} ->
+            MibToHrlOpts =
+                case proplists:get_value(verbosity, AllOpts, undefined) of
+                    undefined ->
+                        #options{specific = [],
+                                 cwd = rebar_dir:get_cwd()};
+                    Verbosity ->
+                        #options{specific = [{verbosity, Verbosity}],
+                                 cwd = rebar_dir:get_cwd()}
+                end,
+            ok = snmpc:mib_to_hrl(Mib, Mib, MibToHrlOpts),
+            rebar_file_utils:mv(HrlFilename, HrlOut),
+            ok;
+        {error, compilation_failed} ->
+            ?FAIL
+    end.
+
+clean(MibFiles, AppInfo) ->
+    AppDir = rebar_app_info:dir(AppInfo),
+    MIBs = [filename:rootname(filename:basename(MIB)) || MIB <- MibFiles],
+    rebar_file_utils:delete_each(
+      [filename:join([AppDir, "include", MIB++".hrl"]) || MIB <- MIBs]),
+    ok = rebar_file_utils:rm_rf(filename:join([AppDir, "priv/mibs/*.bin"])).

src/rebar_compiler_xrl.erl

@@ -0,0 +1,50 @@
+-module(rebar_compiler_xrl).
+
+-behaviour(rebar_compiler).
+
+-export([context/1,
+         needed_files/3,
+         dependencies/3,
+         compile/4,
+         clean/2]).
+
+context(AppInfo) ->
+    Dir = rebar_app_info:dir(AppInfo),
+    Mappings = [{".erl", filename:join([Dir, "src"])}],
+    #{src_dirs => ["src"],
+      include_dirs => [],
+      src_ext => ".xrl",
+      out_mappings => Mappings}.
+
+needed_files(_, FoundFiles, AppInfo) ->
+    FirstFiles = [],
+
+    %% Remove first files from found files
+    RestFiles = [Source || Source <- FoundFiles, not lists:member(Source, FirstFiles)],
+
+    Opts = rebar_opts:get(rebar_app_info:opts(AppInfo), xrl_opts, []),
+
+    {{FirstFiles, Opts}, {RestFiles, Opts}}.
+
+dependencies(_, _, _) ->
+    [].
+
+compile(Source, [{_, OutDir}], _, Opts) ->
+    BaseName = filename:basename(Source),
+    Target = filename:join([OutDir, BaseName]),
+    AllOpts = [{parserfile, Target} | Opts],
+    AllOpts1 = [{includefile, filename:join(OutDir, I)} || {includefile, I} <- AllOpts,
+                                                           filename:pathtype(I) =:= relative],
+    case leex:file(Source, AllOpts1 ++ [{return, true}]) of
+        {ok, _} ->
+            ok;
+        {ok, _Mod, Ws} ->
+            rebar_compiler:ok_tuple(Source, Ws);
+        {error, Es, Ws} ->
+            rebar_compiler:error_tuple(Source, Es, Ws, AllOpts1)
+    end.
+
+clean(XrlFiles, _AppInfo) ->
+    rebar_file_utils:delete_each(
+      [rebar_utils:to_list(re:replace(F, "\\.xrl$", ".erl", [unicode]))
+       || F <- XrlFiles]).

src/rebar_compiler_yrl.erl

@@ -0,0 +1,49 @@
+-module(rebar_compiler_yrl).
+
+-behaviour(rebar_compiler).
+
+-export([context/1,
+         needed_files/3,
+         dependencies/3,
+         compile/4,
+         clean/2]).
+
+context(AppInfo) ->
+    Dir = rebar_app_info:dir(AppInfo),
+    Mappings = [{".erl", filename:join([Dir, "src"])}],
+    #{src_dirs => ["src"],
+      include_dirs => [],
+      src_ext => ".yrl",
+      out_mappings => Mappings}.
+
+needed_files(_, FoundFiles, AppInfo) ->
+    FirstFiles = [],
+
+    %% Remove first files from found files
+    RestFiles = [Source || Source <- FoundFiles, not lists:member(Source, FirstFiles)],
+
+    Opts = rebar_opts:get(rebar_app_info:opts(AppInfo), yrl_opts, []),
+    {{FirstFiles, Opts}, {RestFiles, Opts}}.
+
+dependencies(_, _, _) ->
+    [].
+
+compile(Source, [{_, OutDir}], _, Opts) ->
+    BaseName = filename:basename(Source),
+    Target = filename:join([OutDir, BaseName]),
+    AllOpts = [{parserfile, Target} | Opts],
+    AllOpts1 = [{includefile, filename:join(OutDir, I)} || {includefile, I} <- AllOpts,
+                                                           filename:pathtype(I) =:= relative],
+    case yecc:file(Source, AllOpts1 ++ [{return, true}]) of
+        {ok, _} ->
+            ok;
+        {ok, _Mod, Ws} ->
+            rebar_compiler:ok_tuple(Source, Ws);
+        {error, Es, Ws} ->
+            rebar_compiler:error_tuple(Source, Es, Ws, AllOpts1)
+    end.
+
+clean(YrlFiles, _AppInfo) ->
+    rebar_file_utils:delete_each(
+      [rebar_utils:to_list(re:replace(F, "\\.yrl$", ".erl", [unicode]))
+       || F <- YrlFiles]).

src/rebar_config.erl

@@ -74,7 +74,7 @@ consult_lock_file(File) ->
             read_attrs(beta, Locks, []);
         [{Vsn, Locks}|Attrs] when is_list(Locks) -> % versioned lock file
             %% Because this is the first version of rebar3 to introduce a lock
-            %% file, all versionned lock files with a different versions have
+            %% file, all versioned lock files with a different version have
             %% to be newer.
             case Vsn of
                 ?CONFIG_VERSION ->

src/rebar_dir.erl

@@ -301,9 +301,8 @@ all_src_dirs(Opts, SrcDefault, ExtraDefault) ->
 src_dir_opts(Opts, Dir) ->
     RawSrcDirs = raw_src_dirs(src_dirs, Opts, []),
     RawExtraSrcDirs = raw_src_dirs(extra_src_dirs, Opts, []),
-    AllOpts = [Opt || {D,Opt} <- RawSrcDirs++RawExtraSrcDirs,
-                      D==Dir],
-    lists:ukeysort(1,proplists:unfold(lists:append(AllOpts))).
+    AllOpts = [Opt || {D, Opt} <- RawSrcDirs++RawExtraSrcDirs, D==Dir],
+    lists:ukeysort(1, proplists:unfold(lists:append(AllOpts))).
 
 %%% @doc
 %%% Return the value of the 'recursive' option for the given directory.

src/rebar_erlc_compiler.erl

@@ -92,6 +92,7 @@ compile(AppInfo) when element(1, AppInfo) == app_info_t ->
 %% @doc compile an individual application.
 -spec compile(rebar_app_info:t(), compile_opts()) -> ok.
 compile(AppInfo, CompileOpts) when element(1, AppInfo) == app_info_t ->
+    warn_deprecated(),
     Dir = rebar_utils:to_list(rebar_app_info:out_dir(AppInfo)),
     RebarOpts = rebar_app_info:opts(AppInfo),
 
@@ -99,6 +100,7 @@ compile(AppInfo, CompileOpts) when element(1, AppInfo) == app_info_t ->
                {recursive, dir_recursive(RebarOpts, "src", CompileOpts)}],
     MibsOpts = [check_last_mod,
                 {recursive, dir_recursive(RebarOpts, "mibs", CompileOpts)}],
+
     rebar_base_compiler:run(RebarOpts,
                             check_files([filename:join(Dir, File)
                                          || File <- rebar_opts:get(RebarOpts, xrl_first_files, [])]),
@@ -147,6 +149,7 @@ compile(RebarOpts, BaseDir, OutDir) ->
 compile(State, BaseDir, OutDir, CompileOpts) when element(1, State) == state_t ->
     compile(rebar_state:opts(State), BaseDir, OutDir, CompileOpts);
 compile(RebarOpts, BaseDir, OutDir, CompileOpts) ->
+    warn_deprecated(),
     SrcDirs = lists:map(fun(SrcDir) -> filename:join(BaseDir, SrcDir) end,
                         rebar_dir:src_dirs(RebarOpts, ["src"])),
     compile_dirs(RebarOpts, BaseDir, SrcDirs, OutDir, CompileOpts),
@@ -363,6 +366,7 @@ effects_code_generation(Option) ->
         report_errors -> false;
         return_errors-> false;
         return_warnings-> false;
+        report -> false;
         warnings_as_errors -> false;
         binary -> false;
         verbose -> false;
@@ -802,7 +806,7 @@ valid_erl_first_conf(FileList) ->
     Strs = filter_file_list(FileList),
     case rebar_utils:is_list_of_strings(Strs) of
         true -> true;
-        false -> ?ABORT("An invalid file list (~p) was provided as part of your erl_files_first directive",
+        false -> ?ABORT("An invalid file list (~p) was provided as part of your erl_first_files directive",
                         [FileList])
     end.
 
@@ -823,3 +827,15 @@ atoms_in_erl_first_files_warning(Atoms) ->
       "that you change these entires to string format "
       "(e.g., \"src/module.erl\") ",
   ?WARN(W, [Atoms]).
+
+warn_deprecated() ->
+    case get({deprecate_warn, ?MODULE}) of
+        undefined ->
+            ?WARN("Calling deprecated ~p compiler module. This module has been "
+                  "replaced by rebar_compiler and rebar_compiler_erl, but will "
+                  "remain available.", [?MODULE]),
+            put({deprecate_warn, ?MODULE}, true),
+            ok;
+        _ ->
+            ok
+    end.

src/rebar_fetch.erl

@@ -7,104 +7,74 @@
 %% -------------------------------------------------------------------
 -module(rebar_fetch).
 
--export([lock_source/3,
-         download_source/3,
-         needs_update/3]).
+-export([lock_source/2,
+         download_source/2,
+         needs_update/2]).
 
 -export([format_error/1]).
 
 -include("rebar.hrl").
 -include_lib("providers/include/providers.hrl").
 
--spec lock_source(file:filename_all(), rebar_resource:resource(), rebar_state:t()) ->
-                         rebar_resource:resource() | {error, string()}.
-lock_source(AppDir, Source, State) ->
-    Resources = rebar_state:resources(State),
-    Module = get_resource_type(Source, Resources),
-    Module:lock(AppDir, Source).
+-spec lock_source(rebar_app_info:t(), rebar_state:t())
+                 -> rebar_resource_v2:source() | {error, string()}.
+lock_source(AppInfo, State)      ->
+    rebar_resource_v2:lock(AppInfo, State).
 
--spec download_source(file:filename_all(), rebar_resource:resource(), rebar_state:t()) ->
-                             true | {error, any()}.
-download_source(AppDir, Source, State) ->
-    try download_source_(AppDir, Source, State) of
-        true ->
-            true;
-        Error ->
-            throw(?PRV_ERROR(Error))
+-spec download_source(rebar_app_info:t(), rebar_state:t())
+                     -> rebar_app_info:t() | {error, any()}.
+download_source(AppInfo, State)  ->
+    AppDir = rebar_app_info:dir(AppInfo),
+    try download_source_(AppInfo, State) of
+        ok ->
+            %% freshly downloaded, update the app info opts to reflect the new config
+            Config = rebar_config:consult(AppDir),
+            AppInfo1 = rebar_app_info:update_opts(AppInfo, rebar_app_info:opts(AppInfo), Config),
+            case rebar_app_discover:find_app(AppInfo1, AppDir, all) of
+                {true, AppInfo2} ->
+                    rebar_app_info:is_available(AppInfo2, true);
+                false ->
+                    throw(?PRV_ERROR({dep_app_not_found, rebar_app_info:name(AppInfo1)}))
+            end;
+        {error, Reason} ->
+            throw(?PRV_ERROR(Reason))
     catch
+        throw:{no_resource, Type, Location} ->
+            throw(?PRV_ERROR({no_resource, Location, Type}));
         ?WITH_STACKTRACE(C,T,S)
             ?DEBUG("rebar_fetch exception ~p ~p ~p", [C, T, S]),
-            throw(?PRV_ERROR({fetch_fail, Source}))
+            throw(?PRV_ERROR({fetch_fail, rebar_app_info:source(AppInfo)}))
     end.
 
-download_source_(AppDir, Source, State) ->
-    Resources = rebar_state:resources(State),
-    Module = get_resource_type(Source, Resources),
+download_source_(AppInfo, State) ->
+    AppDir = rebar_app_info:dir(AppInfo),
     TmpDir = ec_file:insecure_mkdtemp(),
     AppDir1 = rebar_utils:to_list(AppDir),
-    case Module:download(TmpDir, Source, State) of
-        {ok, _} ->
+    case rebar_resource_v2:download(TmpDir, AppInfo, State) of
+        ok ->
             ec_file:mkdir_p(AppDir1),
             code:del_path(filename:absname(filename:join(AppDir1, "ebin"))),
             ok = rebar_file_utils:rm_rf(filename:absname(AppDir1)),
             ?DEBUG("Moving checkout ~p to ~p", [TmpDir, filename:absname(AppDir1)]),
-            ok = rebar_file_utils:mv(TmpDir, filename:absname(AppDir1)),
-            true;
+            rebar_file_utils:mv(TmpDir, filename:absname(AppDir1));
         Error ->
             Error
     end.
 
--spec needs_update(file:filename_all(), rebar_resource:resource(), rebar_state:t()) -> boolean() | {error, string()}.
-needs_update(AppDir, Source, State) ->
-    Resources = rebar_state:resources(State),
-    Module = get_resource_type(Source, Resources),
+-spec needs_update(rebar_app_info:t(), rebar_state:t())
+                  -> boolean() | {error, string()}.
+needs_update(AppInfo, State) ->
     try
-        Module:needs_update(AppDir, Source)
+        rebar_resource_v2:needs_update(AppInfo, State)
     catch
         _:_ ->
             true
     end.
 
-format_error({bad_download, CachePath}) ->
-    io_lib:format("Download of package does not match md5sum from server: ~ts", [CachePath]);
-format_error({unexpected_hash, CachePath, Expected, Found}) ->
-    io_lib:format("The checksum for package at ~ts (~ts) does not match the "
-                  "checksum previously locked (~ts). Either unlock or "
-                  "upgrade the package, or make sure you fetched it from "
-                  "the same index from which it was initially fetched.",
-                  [CachePath, Found, Expected]);
-format_error({failed_extract, CachePath}) ->
-    io_lib:format("Failed to extract package: ~ts", [CachePath]);
-format_error({bad_etag, Source}) ->
-    io_lib:format("MD5 Checksum comparison failed for: ~ts", [Source]);
 format_error({fetch_fail, Name, Vsn}) ->
     io_lib:format("Failed to fetch and copy dep: ~ts-~ts", [Name, Vsn]);
 format_error({fetch_fail, Source}) ->
     io_lib:format("Failed to fetch and copy dep: ~p", [Source]);
-format_error({bad_checksum, File}) ->
-    io_lib:format("Checksum mismatch against tarball in ~ts", [File]);
-format_error({bad_registry_checksum, File}) ->
-    io_lib:format("Checksum mismatch against registry in ~ts", [File]).
-
-get_resource_type({Type, Location}, Resources) ->
-    find_resource_module(Type, Location, Resources);
-get_resource_type({Type, Location, _}, Resources) ->
-    find_resource_module(Type, Location, Resources);
-get_resource_type({Type, _, _, Location}, Resources) ->
-    find_resource_module(Type, Location, Resources);
-get_resource_type(_, _) ->
-    rebar_pkg_resource.
-
-find_resource_module(Type, Location, Resources) ->
-    case lists:keyfind(Type, 1, Resources) of
-        false ->
-            case code:which(Type) of
-                non_existing ->
-                    {error, io_lib:format("Cannot handle dependency ~ts.~n"
-                                         "     No module for resource type ~p", [Location, Type])};
-                _ ->
-                    Type
-            end;
-        {Type, Module} ->
-            Module
-    end.
+format_error({dep_app_not_found, AppName}) ->
+    io_lib:format("Dependency failure: source for ~ts does not contain a "
+                  "recognizable project and can not be built", [AppName]).

src/rebar_file_utils.erl

@@ -43,7 +43,8 @@
          path_from_ancestor/2,
          canonical_path/1,
          resolve_link/1,
-         split_dirname/1]).
+         split_dirname/1,
+         ensure_dir/1]).
 
 -include("rebar.hrl").
 
@@ -386,7 +387,7 @@ reset_dir(Path) ->
     %% delete the directory if it exists
     _ = ec_file:remove(Path, [recursive]),
     %% recreate the directory
-    filelib:ensure_dir(filename:join([Path, "dummy.beam"])).
+    ensure_dir(Path).
 
 
 %% Linux touch but using erlang functions to work in bot *nix os and
@@ -440,6 +441,10 @@ resolve_link(Path) ->
 split_dirname(Path) ->
     {filename:dirname(Path), filename:basename(Path)}.
 
+-spec ensure_dir(filelib:dirname_all()) -> ok | {error, file:posix()}.
+ensure_dir(Path) ->
+    filelib:ensure_dir(filename:join(Path, "fake_file")).
+
 %% ===================================================================
 %% Internal functions
 %% ===================================================================
@@ -505,7 +510,7 @@ cp_r_win32({true, SourceDir}, {false, DestDir}) ->
         false ->
             %% Specifying a target directory that doesn't currently exist.
             %% So let's attempt to create this directory
-            case filelib:ensure_dir(filename:join(DestDir, "dummy")) of
+            case ensure_dir(DestDir) of
                 ok ->
                     ok = xcopy_win32(SourceDir, DestDir);
                 {error, Reason} ->

src/rebar_git_resource.erl

@@ -2,21 +2,30 @@
 %% ex: ts=4 sw=4 et
 -module(rebar_git_resource).
 
--behaviour(rebar_resource).
+-behaviour(rebar_resource_v2).
 
--export([lock/2
-        ,download/3
-        ,needs_update/2
-        ,make_vsn/1]).
+-export([init/2,
+         lock/2,
+         download/4,
+         needs_update/2,
+         make_vsn/2]).
 
 -include("rebar.hrl").
 
 %% Regex used for parsing scp style remote url
 -define(SCP_PATTERN, "\\A(?<username>[^@]+)@(?<host>[^:]+):(?<path>.+)\\z").
 
-lock(AppDir, {git, Url, _}) ->
-    lock(AppDir, {git, Url});
-lock(AppDir, {git, Url}) ->
+-spec init(atom(), rebar_state:t()) -> {ok, rebar_resource_v2:resource()}.
+init(Type, _State) ->
+    Resource = rebar_resource_v2:new(Type, ?MODULE, #{}),
+    {ok, Resource}.
+
+lock(AppInfo, _) ->
+    lock_(rebar_app_info:dir(AppInfo), rebar_app_info:source(AppInfo)).
+
+lock_(AppDir, {git, Url, _}) ->
+    lock_(AppDir, {git, Url});
+lock_(AppDir, {git, Url}) ->
     AbortMsg = lists:flatten(io_lib:format("Locking of git dependency failed in ~ts", [AppDir])),
     Dir = rebar_utils:escape_double_quotes(AppDir),
     {ok, VsnString} =
@@ -33,14 +42,17 @@ lock(AppDir, {git, Url}) ->
 
 %% Return true if either the git url or tag/branch/ref is not the same as the currently
 %% checked out git repo for the dep
-needs_update(Dir, {git, Url, {tag, Tag}}) ->
+needs_update(AppInfo, _) ->
+    needs_update_(rebar_app_info:dir(AppInfo), rebar_app_info:source(AppInfo)).
+
+needs_update_(Dir, {git, Url, {tag, Tag}}) ->
     {ok, Current} = rebar_utils:sh(?FMT("git describe --tags --exact-match", []),
                                    [{cd, Dir}]),
     Current1 = rebar_string:trim(rebar_string:trim(Current, both, "\n"),
                                  both, "\r"),
     ?DEBUG("Comparing git tag ~ts with ~ts", [Tag, Current1]),
     not ((Current1 =:= Tag) andalso compare_url(Dir, Url));
-needs_update(Dir, {git, Url, {branch, Branch}}) ->
+needs_update_(Dir, {git, Url, {branch, Branch}}) ->
     %% Fetch remote so we can check if the branch has changed
     SafeBranch = rebar_utils:escape_chars(Branch),
     {ok, _} = rebar_utils:sh(?FMT("git fetch origin ~ts", [SafeBranch]),
@@ -50,9 +62,9 @@ needs_update(Dir, {git, Url, {branch, Branch}}) ->
                                    [{cd, Dir}]),
     ?DEBUG("Checking git branch ~ts for updates", [Branch]),
     not ((Current =:= []) andalso compare_url(Dir, Url));
-needs_update(Dir, {git, Url, "master"}) ->
-    needs_update(Dir, {git, Url, {branch, "master"}});
-needs_update(Dir, {git, _, Ref}) ->
+needs_update_(Dir, {git, Url, "master"}) ->
+    needs_update_(Dir, {git, Url, {branch, "master"}});
+needs_update_(Dir, {git, _, Ref}) ->
     {ok, Current} = rebar_utils:sh(?FMT("git rev-parse --short=7 -q HEAD", []),
                                    [{cd, Dir}]),
     Current1 = rebar_string:trim(rebar_string:trim(Current, both, "\n"),
@@ -98,25 +110,35 @@ parse_git_url(not_scp, Url) ->
             {error, Reason}
     end.
 
-download(Dir, {git, Url}, State) ->
+download(TmpDir, AppInfo, State, _) ->
+    case download_(TmpDir, rebar_app_info:source(AppInfo), State) of
+        {ok, _} ->
+            ok;
+        {error, Reason} ->
+            {error, Reason};
+        Error ->
+            {error, Error}
+    end.
+
+download_(Dir, {git, Url}, State) ->
     ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []),
-    download(Dir, {git, Url, {branch, "master"}}, State);
-download(Dir, {git, Url, ""}, State) ->
+    download_(Dir, {git, Url, {branch, "master"}}, State);
+download_(Dir, {git, Url, ""}, State) ->
     ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []),
-    download(Dir, {git, Url, {branch, "master"}}, State);
-download(Dir, {git, Url, {branch, Branch}}, _State) ->
+    download_(Dir, {git, Url, {branch, "master"}}, State);
+download_(Dir, {git, Url, {branch, Branch}}, _State) ->
     ok = filelib:ensure_dir(Dir),
     maybe_warn_local_url(Url),
     git_clone(branch, git_vsn(), Url, Dir, Branch);
-download(Dir, {git, Url, {tag, Tag}}, _State) ->
+download_(Dir, {git, Url, {tag, Tag}}, _State) ->
     ok = filelib:ensure_dir(Dir),
     maybe_warn_local_url(Url),
     git_clone(tag, git_vsn(), Url, Dir, Tag);
-download(Dir, {git, Url, {ref, Ref}}, _State) ->
+download_(Dir, {git, Url, {ref, Ref}}, _State) ->
     ok = filelib:ensure_dir(Dir),
     maybe_warn_local_url(Url),
     git_clone(ref, git_vsn(), Url, Dir, Ref);
-download(Dir, {git, Url, Rev}, _State) ->
+download_(Dir, {git, Url, Rev}, _State) ->
     ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []),
     ok = filelib:ensure_dir(Dir),
     maybe_warn_local_url(Url),
@@ -201,7 +223,10 @@ git_vsn_fetch() ->
             undefined
     end.
 
-make_vsn(Dir) ->
+make_vsn(AppInfo, _) ->
+    make_vsn_(rebar_app_info:dir(AppInfo)).
+
+make_vsn_(Dir) ->
     case collect_default_refcount(Dir) of
         Vsn={plain, _} ->
             Vsn;

src/rebar_hex_repos.erl

@@ -0,0 +1,142 @@
+-module(rebar_hex_repos).
+
+-export([from_state/2,
+         get_repo_config/2,
+         auth_config/1,
+         update_auth_config/2,
+         format_error/1]).
+
+-ifdef(TEST).
+%% exported for test purposes
+-export([repos/1, merge_repos/1]).
+-endif.
+
+-include("rebar.hrl").
+-include_lib("providers/include/providers.hrl").
+
+-export_type([repo/0]).
+
+-type repo() :: #{name => unicode:unicode_binary(),
+                  api_url => binary(),
+                  api_key => binary(),
+                  repo_url => binary(),
+                  repo_public_key => binary(),
+                  repo_verify => binary()}.
+
+from_state(BaseConfig, State) ->
+    HexConfig = rebar_state:get(State, hex, []),
+    Repos = repos(HexConfig),
+    %% auth is stored in a separate config file since the plugin generates and modifies it
+    Auth = ?MODULE:auth_config(State),
+    %% add base config entries that are specific to use by rebar3 and not overridable
+    Repos1 = merge_with_base_and_auth(Repos, BaseConfig, Auth),
+    %% merge organizations parent repo options into each oraganization repo
+    update_organizations(Repos1).
+
+-spec get_repo_config(unicode:unicode_binary(), rebar_state:t() | [repo()])
+                     -> {ok, repo()} | error.
+get_repo_config(RepoName, Repos) when is_list(Repos) ->
+    case ec_lists:find(fun(#{name := N}) -> N =:= RepoName end, Repos) of
+        error ->
+            throw(?PRV_ERROR({repo_not_found, RepoName}));
+        {ok, RepoConfig} ->
+            {ok, RepoConfig}
+    end;
+get_repo_config(RepoName, State) ->
+    Resources = rebar_state:resources(State),
+    #{repos := Repos} = rebar_resource_v2:find_resource_state(pkg, Resources),
+    get_repo_config(RepoName, Repos).
+
+merge_with_base_and_auth(Repos, BaseConfig, Auth) ->
+    [maps:merge(maps:get(maps:get(name, Repo), Auth, #{}),
+                maps:merge(Repo, BaseConfig)) || Repo <- Repos].
+
+%% A user's list of repos are merged by name while keeping the order
+%% intact. The order is based on the first use of a repo by name in the
+%% list. The default repo is appended to the user's list.
+repos(HexConfig) ->
+    HexDefaultConfig = default_repo(),
+    case [R || R <- HexConfig, element(1, R) =:= repos] of
+        [] ->
+            [HexDefaultConfig];
+        %% we only care if the first element is a replace entry
+        [{repos, replace, Repos} | _]->
+            merge_repos(Repos);
+        Repos ->
+            RepoList = repo_list(Repos),
+            merge_repos(RepoList ++ [HexDefaultConfig])
+    end.
+
+-spec merge_repos([repo()]) -> [repo()].
+merge_repos(Repos) ->
+    lists:foldl(fun(R=#{name := Name}, ReposAcc) ->
+                        %% private organizations include the parent repo before a :
+                        case rebar_string:split(Name, <<":">>) of
+                            [Repo, Org] ->
+                                update_repo_list(R#{name => Name,
+                                                    organization => Org,
+                                                    parent => Repo}, ReposAcc);
+                            _ ->
+                                update_repo_list(R, ReposAcc)
+                        end
+                end, [], Repos).
+
+update_organizations(Repos) ->
+    lists:map(fun(Repo=#{organization := Organization,
+                         parent := ParentName}) ->
+                      {ok, Parent} = get_repo_config(ParentName, Repos),
+                      ParentRepoUrl = rebar_utils:to_list(maps:get(repo_url, Parent)),
+                      {ok, RepoUrl} =
+                          rebar_utils:url_append_path(ParentRepoUrl,
+                                                      filename:join("repos", rebar_utils:to_list(Organization))),
+                      %% still let the organization config override this constructed repo url
+                      maps:merge(Parent#{repo_url => rebar_utils:to_binary(RepoUrl)}, Repo);
+                 (Repo) ->
+                      Repo
+              end, Repos).
+
+update_repo_list(R=#{name := N}, [H=#{name := HN} | Rest]) when N =:= HN ->
+    [maps:merge(R, H) | Rest];
+update_repo_list(R, [H | Rest]) ->
+    [H | update_repo_list(R, Rest)];
+update_repo_list(R, []) ->
+    [R].
+
+default_repo() ->
+    HexDefaultConfig = hex_core:default_config(),
+    HexDefaultConfig#{name => ?PUBLIC_HEX_REPO}.
+
+repo_list([]) ->
+    [];
+repo_list([{repos, Repos} | T]) ->
+    Repos ++ repo_list(T);
+repo_list([{repos, replace, Repos} | T]) ->
+    Repos ++ repo_list(T).
+
+format_error({repo_not_found, RepoName}) ->
+    io_lib:format("The repo ~ts was not found in the configuration.", [RepoName]).
+
+%% auth functions
+
+%% authentication is in a separate config file because the hex plugin updates it
+
+-spec auth_config_file(rebar_state:t()) -> file:filename_all().
+auth_config_file(State) ->
+    filename:join(rebar_dir:global_config_dir(State), ?HEX_AUTH_FILE).
+
+-spec auth_config(rebar_state:t()) -> map().
+auth_config(State) ->
+    case file:consult(auth_config_file(State)) of
+        {ok, [Config]} ->
+            Config;
+        _ ->
+            #{}
+    end.
+
+-spec update_auth_config(map(), rebar_state:t()) -> ok.
+update_auth_config(Updates, State) ->
+    Config = auth_config(State),
+    AuthConfigFile = auth_config_file(State),
+    ok = filelib:ensure_dir(AuthConfigFile),
+    NewConfig = iolist_to_binary([io_lib:print(maps:merge(Config, Updates)) | ".\n"]),
+    ok = file:write_file(AuthConfigFile, NewConfig).

src/rebar_hg_resource.erl

@@ -2,39 +2,51 @@
 %% ex: ts=4 sw=4 et
 -module(rebar_hg_resource).
 
--behaviour(rebar_resource).
+-behaviour(rebar_resource_v2).
 
--export([lock/2
-        ,download/3
-        ,needs_update/2
-        ,make_vsn/1]).
+-export([init/2,
+         lock/2,
+         download/4,
+         needs_update/2,
+         make_vsn/2]).
 
 -include("rebar.hrl").
 
-lock(AppDir, {hg, Url, _}) ->
-    lock(AppDir, {hg, Url});
-lock(AppDir, {hg, Url}) ->
+-spec init(atom(), rebar_state:t()) -> {ok, rebar_resource_v2:resource()}.
+init(Type, _State) ->
+    Resource = rebar_resource_v2:new(Type, ?MODULE, #{}),
+    {ok, Resource}.
+
+lock(AppInfo, _) ->
+    lock_(rebar_app_info:dir(AppInfo), rebar_app_info:source(AppInfo)).
+
+lock_(AppDir, {hg, Url, _}) ->
+    lock_(AppDir, {hg, Url});
+lock_(AppDir, {hg, Url}) ->
     Ref = get_ref(AppDir),
     {hg, Url, {ref, Ref}}.
 
 %% Return `true' if either the hg url or tag/branch/ref is not the same as
 %% the currently checked out repo for the dep
-needs_update(Dir, {hg, Url, {tag, Tag}}) ->
+needs_update(AppInfo, _) ->
+    needs_update_(rebar_app_info:dir(AppInfo), rebar_app_info:source(AppInfo)).
+
+needs_update_(Dir, {hg, Url, {tag, Tag}}) ->
     Ref = get_ref(Dir),
     {ClosestTag, Distance} = get_tag_distance(Dir, Ref),
     ?DEBUG("Comparing hg tag ~ts with ref ~ts (closest tag is ~ts at distance ~ts)",
            [Tag, Ref, ClosestTag, Distance]),
     not ((Distance =:= "0") andalso (Tag =:= ClosestTag)
          andalso compare_url(Dir, Url));
-needs_update(Dir, {hg, Url, {branch, Branch}}) ->
+needs_update_(Dir, {hg, Url, {branch, Branch}}) ->
     Ref = get_ref(Dir),
     BRef = get_branch_ref(Dir, Branch),
     not ((Ref =:= BRef) andalso compare_url(Dir, Url));
-needs_update(Dir, {hg, Url, "default"}) ->
+needs_update_(Dir, {hg, Url, "default"}) ->
     Ref = get_ref(Dir),
     BRef = get_branch_ref(Dir, "default"),
     not ((Ref =:= BRef) andalso compare_url(Dir, Url));
-needs_update(Dir, {hg, Url, Ref}) ->
+needs_update_(Dir, {hg, Url, Ref}) ->
     LocalRef = get_ref(Dir),
     TargetRef = case Ref of
         {ref, Ref1} ->
@@ -48,13 +60,23 @@ needs_update(Dir, {hg, Url, Ref}) ->
     ?DEBUG("Comparing hg ref ~ts with ~ts", [Ref1, LocalRef]),
     not ((LocalRef =:= TargetRef) andalso compare_url(Dir, Url)).
 
-download(Dir, {hg, Url}, State) ->
+download(TmpDir, AppInfo, State, _) ->
+    case download_(TmpDir, rebar_app_info:source(AppInfo), State) of
+        {ok, _} ->
+            ok;
+        {error, Reason} ->
+            {error, Reason};
+        Error ->
+            {error, Error}
+    end.
+
+download_(Dir, {hg, Url}, State) ->
     ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []),
-    download(Dir, {hg, Url, {branch, "default"}}, State);
-download(Dir, {hg, Url, ""}, State) ->
+    download_(Dir, {hg, Url, {branch, "default"}}, State);
+download_(Dir, {hg, Url, ""}, State) ->
     ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []),
-    download(Dir, {hg, Url, {branch, "default"}}, State);
-download(Dir, {hg, Url, {branch, Branch}}, _State) ->
+    download_(Dir, {hg, Url, {branch, "default"}}, State);
+download_(Dir, {hg, Url, {branch, Branch}}, _State) ->
     ok = filelib:ensure_dir(Dir),
     maybe_warn_local_url(Url),
     rebar_utils:sh(?FMT("hg clone -q -b ~ts ~ts ~ts",
@@ -62,7 +84,7 @@ download(Dir, {hg, Url, {branch, Branch}}, _State) ->
                         rebar_utils:escape_chars(Url),
                         rebar_utils:escape_chars(filename:basename(Dir))]),
                    [{cd, filename:dirname(Dir)}]);
-download(Dir, {hg, Url, {tag, Tag}}, _State) ->
+download_(Dir, {hg, Url, {tag, Tag}}, _State) ->
     ok = filelib:ensure_dir(Dir),
     maybe_warn_local_url(Url),
     rebar_utils:sh(?FMT("hg clone -q -u ~ts ~ts ~ts",
@@ -70,7 +92,7 @@ download(Dir, {hg, Url, {tag, Tag}}, _State) ->
                          rebar_utils:escape_chars(Url),
                          rebar_utils:escape_chars(filename:basename(Dir))]),
                    [{cd, filename:dirname(Dir)}]);
-download(Dir, {hg, Url, {ref, Ref}}, _State) ->
+download_(Dir, {hg, Url, {ref, Ref}}, _State) ->
     ok = filelib:ensure_dir(Dir),
     maybe_warn_local_url(Url),
     rebar_utils:sh(?FMT("hg clone -q -r ~ts ~ts ~ts",
@@ -78,7 +100,7 @@ download(Dir, {hg, Url, {ref, Ref}}, _State) ->
                          rebar_utils:escape_chars(Url),
                          rebar_utils:escape_chars(filename:basename(Dir))]),
                    [{cd, filename:dirname(Dir)}]);
-download(Dir, {hg, Url, Rev}, _State) ->
+download_(Dir, {hg, Url, Rev}, _State) ->
     ok = filelib:ensure_dir(Dir),
     maybe_warn_local_url(Url),
     rebar_utils:sh(?FMT("hg clone -q -r ~ts ~ts ~ts",
@@ -87,7 +109,10 @@ download(Dir, {hg, Url, Rev}, _State) ->
                          rebar_utils:escape_chars(filename:basename(Dir))]),
                    [{cd, filename:dirname(Dir)}]).
 
-make_vsn(Dir) ->
+make_vsn(AppInfo, _) ->
+    make_vsn_(rebar_app_info:dir(AppInfo)).
+
+make_vsn_(Dir) ->
     BaseHg = "hg -R \"" ++ rebar_utils:escape_double_quotes(Dir) ++ "\" ",
     Ref = get_ref(Dir),
     Cmd = BaseHg ++ "log --template \"{latesttag}+build.{latesttagdistance}.rev.{node|short}\""

src/rebar_hooks.erl

@@ -42,8 +42,7 @@ run_provider_hooks_(Dir, Type, Command, Providers, TypeHooks, State) ->
         [] ->
             State;
         HookProviders ->
-            PluginDepsPaths = lists:usort(rebar_state:code_paths(State, all_plugin_deps)),
-            code:add_pathsa(PluginDepsPaths),
+            rebar_paths:set_paths([plugins], State),
             Providers1 = rebar_state:providers(State),
             State1 = rebar_state:providers(rebar_state:dir(State, Dir), Providers++Providers1),
             case rebar_core:do(HookProviders, State1) of
@@ -51,7 +50,7 @@ run_provider_hooks_(Dir, Type, Command, Providers, TypeHooks, State) ->
                     ?DEBUG(format_error({bad_provider, Type, Command, ProviderName}), []),
                     throw(?PRV_ERROR({bad_provider, Type, Command, ProviderName}));
                 {ok, State2} ->
-                    rebar_utils:remove_from_code_path(PluginDepsPaths),
+                    rebar_paths:set_paths([deps], State2),
                     State2
             end
     end.

src/rebar_otp_app.erl

@@ -59,8 +59,9 @@ compile(State, App) ->
 
 format_error({missing_app_file, Filename}) ->
     io_lib:format("App file is missing: ~ts", [Filename]);
-format_error({file_read, File, Reason}) ->
-    io_lib:format("Failed to read required file ~ts for processing: ~ts", [File, file:format_error(Reason)]);
+format_error({file_read, AppName, File, Reason}) ->
+    io_lib:format("Failed to read required ~ts file for processing the application '~ts': ~ts",
+                  [File, AppName, file:format_error(Reason)]);
 format_error({invalid_name, File, AppName}) ->
     io_lib:format("Invalid ~ts: name of application (~p) must match filename.", [File, AppName]).
 
@@ -79,7 +80,7 @@ validate_app(State, App) ->
                     Error
             end;
         {error, Reason} ->
-            ?PRV_ERROR({file_read, AppFile, Reason})
+            ?PRV_ERROR({file_read, rebar_app_info:name(App), ".app", Reason})
     end.
 
 validate_app_modules(State, App, AppData) ->
@@ -110,7 +111,7 @@ preprocess(State, AppInfo, AppSrcFile) ->
             A1 = apply_app_vars(AppVars, AppData),
 
             %% AppSrcFile may contain instructions for generating a vsn number
-            Vsn = app_vsn(AppData, AppSrcFile, State),
+            Vsn = app_vsn(AppInfo, AppData, AppSrcFile, State),
             A2 = lists:keystore(vsn, 1, A1, {vsn, Vsn}),
 
             %% systools:make_relup/4 fails with {missing_param, registered}
@@ -125,13 +126,13 @@ preprocess(State, AppInfo, AppSrcFile) ->
 
             %% Setup file .app filename and write new contents
             EbinDir = rebar_app_info:ebin_dir(AppInfo),
-            filelib:ensure_dir(filename:join(EbinDir, "dummy.beam")),
+            rebar_file_utils:ensure_dir(EbinDir),
             AppFile = rebar_app_utils:app_src_to_app(OutDir, AppSrcFile),
             ok = rebar_file_utils:write_file_if_contents_differ(AppFile, Spec, utf8),
 
             AppFile;
         {error, Reason} ->
-            throw(?PRV_ERROR({file_read, AppSrcFile, Reason}))
+            throw(?PRV_ERROR({file_read, rebar_app_info:name(AppInfo), ".app.src", Reason}))
     end.
 
 load_app_vars(State) ->
@@ -163,32 +164,17 @@ validate_name(AppName, File) ->
 
 ebin_modules(AppInfo, Dir) ->
     Beams = lists:sort(rebar_utils:beams(filename:join(Dir, "ebin"))),
-    ExtraDirs = extra_dirs(AppInfo),
-    F = fun(Beam) -> not in_extra_dir(AppInfo, Beam, ExtraDirs) end,
-    Filtered = lists:filter(F, Beams),
+    SrcDirs = rebar_dir:src_dirs(rebar_app_info:opts(AppInfo), ["src"]),
+    FindSourceRules = [{".beam", ".erl",
+                        [{"ebin", SrcDir} || SrcDir <- SrcDirs]}],
+    Filtered = lists:filter(fun(Beam) ->
+                                rebar_utils:find_source(filename:basename(Beam),
+                                                        filename:dirname(Beam),
+                                                        FindSourceRules)
+                                        =/= {error, not_found}
+                            end, Beams),
     [rebar_utils:beam_to_mod(N) || N <- Filtered].
 
-extra_dirs(State) ->
-    Extras = rebar_dir:extra_src_dirs(rebar_app_info:opts(State)),
-    SrcDirs = rebar_dir:src_dirs(rebar_app_info:opts(State), ["src"]),
-    %% remove any dirs that are defined in `src_dirs` from `extra_src_dirs`
-    Extras -- SrcDirs.
-
-in_extra_dir(AppInfo, Beam, Dirs) ->
-    lists:any(fun(Dir) -> lists:prefix(filename:join([rebar_app_info:out_dir(AppInfo), Dir]),
-                                       beam_src(Beam)) end,
-              Dirs).
-
-beam_src(Beam) ->
-    case beam_lib:chunks(Beam, [compile_info]) of
-        {ok, {_mod, Chunks}} ->
-            CompileInfo = proplists:get_value(compile_info, Chunks, []),
-            proplists:get_value(source, CompileInfo, []);
-        {error, beam_lib, Reason} ->
-            ?WARN("Couldn't read debug info from ~p for reason: ~p", [Beam, Reason]),
-            []
-    end.
-
 ensure_registered(AppData) ->
     case lists:keyfind(registered, 1, AppData) of
         false ->
@@ -226,10 +212,8 @@ consult_app_file(Filename) ->
             end
     end.
 
-app_vsn(AppData, AppFile, State) ->
-    AppDir = filename:dirname(filename:dirname(AppFile)),
-    Resources = rebar_state:resources(State),
-    rebar_utils:vcs_vsn(get_value(vsn, AppData, AppFile), AppDir, Resources).
+app_vsn(AppInfo, AppData, AppFile, State) ->
+    rebar_utils:vcs_vsn(AppInfo, get_value(vsn, AppData, AppFile), State).
 
 get_value(Key, AppInfo, AppFile) ->
     case proplists:get_value(Key, AppInfo) of

src/rebar_packages.erl

@@ -1,18 +1,18 @@
 -module(rebar_packages).
 
--export([packages/1
-        ,close_packages/0
-        ,load_and_verify_version/1
-        ,deps/3
+-export([get/2
+        ,get_all_names/1
         ,registry_dir/1
-        ,package_dir/1
-        ,registry_checksum/2
-        ,find_highest_matching/6
-        ,find_highest_matching/4
-        ,find_highest_matching_/6
-        ,find_all/3
+        ,package_dir/2
+        ,find_highest_matching/5
         ,verify_table/1
-        ,format_error/1]).
+        ,format_error/1
+        ,update_package/3
+        ,resolve_version/5]).
+
+-ifdef(TEST).
+-export([new_package_table/0, find_highest_matching_/5, cmp_/4, cmpl_/4, valid_vsn/1]).
+-endif.
 
 -export_type([package/0]).
 
@@ -23,119 +23,131 @@
 -type vsn() :: binary().
 -type package() :: pkg_name() | {pkg_name(), vsn()}.
 
--spec packages(rebar_state:t()) -> ets:tid().
-packages(State) ->
-    catch ets:delete(?PACKAGE_TABLE),
-    case load_and_verify_version(State) of
-        true ->
-            ok;
-        false ->
-            ?DEBUG("Error loading package index.", []),
-            handle_bad_index(State)
+format_error({missing_package, Name, Vsn}) ->
+    io_lib:format("Package not found in any repo: ~ts ~ts", [rebar_utils:to_binary(Name),
+                                                             rebar_utils:to_binary(Vsn)]);
+format_error({missing_package, Pkg}) ->
+    io_lib:format("Package not found in any repo: ~p", [Pkg]).
+
+-spec get(rebar_hex_repos:repo(), binary()) -> {ok, map()} | {error, term()}.
+get(Config, Name) ->
+    try hex_api_package:get(Config, Name) of
+        {ok, {200, _Headers, PkgInfo}} ->
+            {ok, PkgInfo};
+        {ok, {404, _, _}} ->
+            {error, not_found};
+        Error ->
+            ?DEBUG("Hex api request failed: ~p", [Error]),
+            {error, unknown}
+    catch
+        error:{badmatch, {error, {failed_connect, _}}} ->
+            {error, failed_to_connect};
+        _:Exception ->
+            ?DEBUG("hex_api_package:get failed: ~p", [Exception]),
+            {error, unknown}
     end.
 
-handle_bad_index(State) ->
-    ?ERROR("Bad packages index. Trying to fix by updating the registry.", []),
-    {ok, State1} = rebar_prv_update:do(State),
-    case load_and_verify_version(State1) of
-        true ->
-            ok;
-        false ->
-            %% Still unable to load after an update, create an empty registry
-            ets:new(?PACKAGE_TABLE, [named_table, public])
+
+-spec get_all_names(rebar_state:t()) -> [binary()].
+get_all_names(State) ->    
+    verify_table(State),
+    lists:usort(ets:select(?PACKAGE_TABLE, [{#package{key={'$1', '_', '_'},
+                                                      _='_'}, 
+                                             [], ['$1']}])).
+
+-spec get_package_versions(unicode:unicode_binary(), ec_semver:semver(),
+                           unicode:unicode_binary(),
+                           ets:tid(), rebar_state:t()) -> [vsn()].
+get_package_versions(Dep, {_, AlphaInfo}, Repo, Table, State) ->
+    ?MODULE:verify_table(State),
+    AllowPreRelease = rebar_state:get(State, deps_allow_prerelease, false)
+        orelse AlphaInfo =/= {[],[]},
+    ets:select(Table, [{#package{key={Dep, {'$1', '$2'}, Repo},
+                                 _='_'},
+                        [{'==', '$2', {{[],[]}}} || not AllowPreRelease], [{{'$1', '$2'}}]}]).
+
+-spec get_package(unicode:unicode_binary(), unicode:unicode_binary(),
+                  binary() | undefined | '_',
+                  [unicode:unicode_binary()] | ['_'], ets:tab(), rebar_state:t())
+                 -> {ok, #package{}} | not_found.
+get_package(Dep, Vsn, undefined, Repos, Table, State) ->
+    get_package(Dep, Vsn, '_', Repos, Table, State);
+get_package(Dep, Vsn, Hash, Repos, Table, State) ->
+    ?MODULE:verify_table(State),
+    case ets:select(Table, [{#package{key={Dep, ec_semver:parse(Vsn), Repo},
+                                      checksum=Hash,
+                                      _='_'}, [], ['$_']} || Repo <- Repos]) of
+        %% have to allow multiple matches in the list for cases that Repo is `_`
+        [Package | _] ->
+            {ok, Package};
+        _ ->
+            not_found
     end.
 
-close_packages() ->
-    catch ets:delete(?PACKAGE_TABLE).
+new_package_table() ->
+    ?PACKAGE_TABLE = ets:new(?PACKAGE_TABLE, [named_table, public, ordered_set, {keypos, 2}]),
+    ets:insert(?PACKAGE_TABLE, {?PACKAGE_INDEX_VERSION, package_index_version}).
 
 load_and_verify_version(State) ->
     {ok, RegistryDir} = registry_dir(State),
     case ets:file2tab(filename:join(RegistryDir, ?INDEX_FILE)) of
         {ok, _} ->
-            case ets:lookup_element(?PACKAGE_TABLE, package_index_version, 2) of
+            case ets:lookup_element(?PACKAGE_TABLE, package_index_version, 1) of
                 ?PACKAGE_INDEX_VERSION ->
                     true;
-                _ ->
+                V ->
+                    %% no reason to confuse the user since we just start fresh and they
+                    %% shouldn't notice, so log as a debug message only
+                    ?DEBUG("Package index version mismatch. Current version ~p, this rebar3 expecting ~p",
+                           [V, ?PACKAGE_INDEX_VERSION]),
                     (catch ets:delete(?PACKAGE_TABLE)),
-                    rebar_prv_update:hex_to_index(State)
+                    new_package_table()                    
             end;
-        _ ->
-            rebar_prv_update:hex_to_index(State)
+        _ ->            
+            new_package_table()
     end.
 
-deps(Name, Vsn, State) ->
-    try
-        deps_(Name, Vsn, State)
-    catch
-        _:_ ->
-            handle_missing_package({Name, Vsn}, State, fun(State1) -> deps_(Name, Vsn, State1) end)
-    end.
-
-deps_(Name, Vsn, State) ->
-    ?MODULE:verify_table(State),
-    ets:lookup_element(?PACKAGE_TABLE, {rebar_utils:to_binary(Name), rebar_utils:to_binary(Vsn)}, 2).
+handle_missing_package(PkgKey, Repo, State, Fun) ->
+    Name = 
+        case PkgKey of
+            {N, Vsn, _Repo} ->
+                ?DEBUG("Package ~ts-~ts not found. Fetching registry updates for "
+                       "package and trying again...", [N, Vsn]),
+                N;
+            _ ->
+                ?DEBUG("Package ~p not found. Fetching registry updates for "
+                       "package and trying again...", [PkgKey]),
+                PkgKey
+        end,
 
-handle_missing_package(Dep, State, Fun) ->
-    case Dep of
-        {Name, Vsn} ->
-            ?INFO("Package ~ts-~ts not found. Fetching registry updates and trying again...", [Name, Vsn]);
-        _ ->
-            ?INFO("Package ~p not found. Fetching registry updates and trying again...", [Dep])
-    end,
-
-    {ok, State1} = rebar_prv_update:do(State),
-    try
-        Fun(State1)
+    update_package(Name, Repo, State),
+    try 
+        Fun(State) 
     catch
         _:_ ->
             %% Even after an update the package is still missing, time to error out
-            throw(?PRV_ERROR({missing_package, Dep}))
+            throw(?PRV_ERROR({missing_package, PkgKey}))
     end.
 
 registry_dir(State) ->
     CacheDir = rebar_dir:global_cache_dir(rebar_state:opts(State)),
-    case rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN) of
-        ?DEFAULT_CDN ->
-            RegistryDir = filename:join([CacheDir, "hex", "default"]),
-            case filelib:ensure_dir(filename:join(RegistryDir, "placeholder")) of
-                ok -> ok;
-                {error, Posix} when Posix == eaccess; Posix == enoent ->
-                    ?ABORT("Could not write to ~p. Please ensure the path is writeable.",
-                           [RegistryDir])
-            end,
-            {ok, RegistryDir};
-        CDN ->
-            case rebar_utils:url_append_path(CDN, ?REMOTE_PACKAGE_DIR) of
-                {ok, Parsed} ->
-                    {ok, {_, _, Host, _, Path, _}} = http_uri:parse(Parsed),
-                    CDNHostPath = lists:reverse(rebar_string:lexemes(Host, ".")),
-                    CDNPath = tl(filename:split(Path)),
-                    RegistryDir = filename:join([CacheDir, "hex"] ++ CDNHostPath ++ CDNPath),
-                    ok = filelib:ensure_dir(filename:join(RegistryDir, "placeholder")),
-                    {ok, RegistryDir};
-                _ ->
-                    {uri_parse_error, CDN}
-            end
-    end.
+    RegistryDir = filename:join([CacheDir, "hex"]),
+    case filelib:ensure_dir(filename:join(RegistryDir, "placeholder")) of
+        ok -> ok;
+        {error, Posix} when Posix == eaccess; Posix == enoent ->
+            ?ABORT("Could not write to ~p. Please ensure the path is writeable.",
+                   [RegistryDir])
+    end,
+    {ok, RegistryDir}.
 
-package_dir(State) ->
-    case registry_dir(State) of
-        {ok, RegistryDir} ->
-            PackageDir = filename:join([RegistryDir, "packages"]),
-            ok = filelib:ensure_dir(filename:join(PackageDir, "placeholder")),
-            {ok, PackageDir};
-        Error ->
-            Error
-    end.
+-spec package_dir(rebar_hex_repos:repo(), rebar_state:t()) -> {ok, file:filename_all()}.
+package_dir(Repo, State) ->
+    {ok, RegistryDir} = registry_dir(State),
+    RepoName = maps:get(name, Repo),
+    PackageDir = filename:join([RegistryDir, rebar_utils:to_list(RepoName), "packages"]),
+    ok = filelib:ensure_dir(filename:join(PackageDir, "placeholder")),
+    {ok, PackageDir}.
 
-registry_checksum({pkg, Name, Vsn, _Hash}, State) ->
-    try
-        ?MODULE:verify_table(State),
-        ets:lookup_element(?PACKAGE_TABLE, {Name, Vsn}, 3)
-    catch
-        _:_ ->
-            throw(?PRV_ERROR({missing_package, rebar_utils:to_binary(Name), rebar_utils:to_binary(Vsn)}))
-    end.
 
 %% Hex supports use of ~> to specify the version required for a dependency.
 %% Since rebar3 requires exact versions to choose from we find the highest
@@ -152,31 +164,28 @@ registry_checksum({pkg, Name, Vsn, _Hash}, State) ->
 %% `~> 2.1.3-dev` | `>= 2.1.3-dev and < 2.2.0`
 %% `~> 2.0` | `>= 2.0.0 and < 3.0.0`
 %% `~> 2.1` | `>= 2.1.0 and < 3.0.0`
-find_highest_matching(Dep, Constraint, Table, State) ->
-    find_highest_matching(undefined, undefined, Dep, Constraint, Table, State).
-
-find_highest_matching(Pkg, PkgVsn, Dep, Constraint, Table, State) ->
-    try find_highest_matching_(Pkg, PkgVsn, Dep, Constraint, Table, State) of
+find_highest_matching(Dep, Constraint, Repo, Table, State) ->
+    try find_highest_matching_(Dep, Constraint, Repo, Table, State) of
         none ->
-            handle_missing_package(Dep, State,
+            handle_missing_package(Dep, Repo, State,
                                    fun(State1) ->
-                                       find_highest_matching_(Pkg, PkgVsn, Dep, Constraint, Table, State1)
+                                       find_highest_matching_(Dep, Constraint, Repo, Table, State1)
                                    end);
         Result ->
             Result
     catch
         _:_ ->
-            handle_missing_package(Dep, State,
+            handle_missing_package(Dep, Repo, State,
                                    fun(State1) ->
-                                       find_highest_matching_(Pkg, PkgVsn, Dep, Constraint, Table, State1)
+                                       find_highest_matching_(Dep, Constraint, Repo, Table, State1)
                                    end)
     end.
 
-find_highest_matching_(Pkg, PkgVsn, Dep, Constraint, Table, State) ->
-    try find_all(Dep, Table, State) of
-        {ok, [Vsn]} ->
-            handle_single_vsn(Pkg, PkgVsn, Dep, Vsn, Constraint);
-        {ok, Vsns} ->
+find_highest_matching_(Dep, Constraint, #{name := Repo}, Table, State) ->
+    try get_package_versions(Dep, Constraint, Repo, Table, State) of
+        [Vsn] ->
+            handle_single_vsn(Vsn, Constraint);
+        Vsns ->
             case handle_vsns(Constraint, Vsns) of
                 none ->
                     none;
@@ -188,18 +197,6 @@ find_highest_matching_(Pkg, PkgVsn, Dep, Constraint, Table, State) ->
             none
     end.
 
-find_all(Dep, Table, State) ->
-    ?MODULE:verify_table(State),
-    try ets:lookup_element(Table, Dep, 2) of
-        [Vsns] when is_list(Vsns)->
-            {ok, Vsns};
-        Vsns ->
-            {ok, Vsns}
-    catch
-        error:badarg ->
-            none
-    end.
-
 handle_vsns(Constraint, Vsns) ->
     lists:foldl(fun(Version, Highest) ->
                         case ec_semver:pes(Version, Constraint) andalso
@@ -211,26 +208,227 @@ handle_vsns(Constraint, Vsns) ->
                         end
                 end, none, Vsns).
 
-handle_single_vsn(Pkg, PkgVsn, Dep, Vsn, Constraint) ->
+handle_single_vsn(Vsn, Constraint) ->
     case ec_semver:pes(Vsn, Constraint) of
         true ->
             {ok, Vsn};
         false ->
-            case {Pkg, PkgVsn} of
-                {undefined, undefined} ->
-                    ?DEBUG("Only existing version of ~ts is ~ts which does not match constraint ~~> ~ts. "
-                          "Using anyway, but it is not guaranteed to work.", [Dep, Vsn, Constraint]);
-                _ ->
-                    ?DEBUG("[~ts:~ts] Only existing version of ~ts is ~ts which does not match constraint ~~> ~ts. "
-                          "Using anyway, but it is not guaranteed to work.", [Pkg, PkgVsn, Dep, Vsn, Constraint])
-            end,
-            {ok, Vsn}
+            none
     end.
 
-format_error({missing_package, Name, Vsn}) ->
-    io_lib:format("Package not found in registry: ~ts-~ts.", [rebar_utils:to_binary(Name), rebar_utils:to_binary(Vsn)]);
-format_error({missing_package, Dep}) ->
-    io_lib:format("Package not found in registry: ~p.", [Dep]).
-
 verify_table(State) ->
     ets:info(?PACKAGE_TABLE, named_table) =:= true orelse load_and_verify_version(State).
+
+parse_deps(Deps) ->
+    [{maps:get(app, D, Name), {pkg, Name, Constraint, undefined}} 
+     || D=#{package := Name,
+            requirement := Constraint} <- Deps].
+
+parse_checksum(<<Checksum:256/big-unsigned>>) ->
+    list_to_binary(
+      rebar_string:uppercase(
+        lists:flatten(io_lib:format("~64.16.0b", [Checksum]))));
+parse_checksum(Checksum) ->
+    Checksum.
+
+update_package(Name, RepoConfig=#{name := Repo}, State) ->
+    ?MODULE:verify_table(State),
+    try hex_repo:get_package(RepoConfig#{repo_key => maps:get(read_key, RepoConfig, <<>>)}, Name) of
+        {ok, {200, _Headers, #{releases := Releases}}} ->
+            _ = insert_releases(Name, Releases, Repo, ?PACKAGE_TABLE),
+            {ok, RegistryDir} = rebar_packages:registry_dir(State),
+            PackageIndex = filename:join(RegistryDir, ?INDEX_FILE),
+            ok = ets:tab2file(?PACKAGE_TABLE, PackageIndex);
+        {ok, {403, _Headers, <<>>}} ->
+            not_found;
+        {ok, {404, _Headers, _}} ->
+            not_found;
+        Error ->
+            ?DEBUG("Hex get_package request failed: ~p", [Error]),
+            %% TODO: add better log message. hex_core should export a format_error
+            ?WARN("Failed to update package from repo ~ts", [Repo]),
+            fail
+    catch
+        _:Exception ->
+            ?DEBUG("hex_repo:get_package failed for package ~p: ~p", [Name, Exception]),
+            fail
+    end.
+
+insert_releases(Name, Releases, Repo, Table) ->
+    [true = ets:insert(Table,
+                       #package{key={Name, ec_semver:parse(Version), Repo},
+                                checksum=parse_checksum(Checksum),
+                                retired=maps:get(retired, Release, false),
+                                dependencies=parse_deps(Dependencies)})
+     || Release=#{checksum := Checksum,
+                  version := Version,
+                  dependencies := Dependencies} <- Releases].
+
+-spec resolve_version(unicode:unicode_binary(), unicode:unicode_binary() | undefined,
+                      binary() | undefined,
+                      ets:tab(), rebar_state:t())
+                     -> {error, {invalid_vsn, unicode:unicode_binary()}} |
+                        not_found |
+                        {ok, #package{}, map()}.
+%% if checksum is defined search for any matching repo matching pkg-vsn and checksum
+resolve_version(Dep, DepVsn, Hash, HexRegistry, State) when is_binary(Hash) ->
+    Resources = rebar_state:resources(State),
+    #{repos := RepoConfigs} = rebar_resource_v2:find_resource_state(pkg, Resources),
+    RepoNames = [RepoName || #{name := RepoName} <- RepoConfigs],
+
+    %% allow retired packages when we have a checksum
+    case get_package(Dep, DepVsn, Hash, RepoNames, HexRegistry, State) of
+        {ok, Package=#package{key={_, _, RepoName}}} ->
+            {ok, RepoConfig} = rebar_hex_repos:get_repo_config(RepoName, RepoConfigs),
+            {ok, Package, RepoConfig};
+        _ ->
+            Fun = fun(Repo) ->
+                      case resolve_version_(Dep, DepVsn, Repo, HexRegistry, State) of
+                          none ->
+                              not_found;
+                          {ok, Vsn} ->
+                              get_package(Dep, Vsn, Hash, [Repo], HexRegistry, State)
+                      end
+                  end,
+            handle_missing_no_exception(Fun, Dep, State)
+    end;
+resolve_version(Dep, undefined, Hash, HexRegistry, State) ->
+    Fun = fun(Repo) ->
+              case highest_matching(Dep, {0,{[],[]}}, Repo, HexRegistry, State) of
+                  none ->
+                      not_found;
+                  {ok, Vsn} ->
+                      get_package(Dep, Vsn, Hash, [Repo], HexRegistry, State)
+              end
+          end,
+    handle_missing_no_exception(Fun, Dep, State);
+resolve_version(Dep, DepVsn, Hash, HexRegistry, State) ->
+    case valid_vsn(DepVsn) of
+        false ->
+            {error, {invalid_vsn, DepVsn}};
+        _ ->
+            Fun = fun(Repo) ->
+                      case resolve_version_(Dep, DepVsn, Repo, HexRegistry, State) of
+                          none ->
+                              not_found;
+                          {ok, Vsn} ->
+                              get_package(Dep, Vsn, Hash, [Repo], HexRegistry, State)
+                      end
+                  end,
+            handle_missing_no_exception(Fun, Dep, State)
+    end.
+
+check_all_repos(Fun, RepoConfigs) ->
+    ec_lists:search(fun(#{name := R}) ->
+                            Fun(R)
+                    end, RepoConfigs).
+
+handle_missing_no_exception(Fun, Dep, State) ->
+    Resources = rebar_state:resources(State),
+    #{repos := RepoConfigs} = rebar_resource_v2:find_resource_state(pkg, Resources),
+
+    %% first check all repos in order for a local match
+    %% if none is found then we step through checking after updating the repo registry
+    case check_all_repos(Fun, RepoConfigs) of
+        not_found ->
+            ec_lists:search(fun(Config=#{name := R}) ->
+                                    case ?MODULE:update_package(Dep, Config, State) of
+                                        ok ->
+                                            Fun(R);
+                                        _ ->
+                                            not_found
+                                    end
+                            end, RepoConfigs);
+        Result ->
+            Result
+    end.
+
+resolve_version_(Dep, DepVsn, Repo, HexRegistry, State) ->
+    case DepVsn of
+        <<"~>", Vsn/binary>> ->
+            highest_matching(Dep, rm_ws(Vsn), Repo, HexRegistry, State);
+        <<">=", Vsn/binary>> ->
+            cmp(Dep, rm_ws(Vsn), Repo, HexRegistry, State, fun ec_semver:gte/2);
+        <<">", Vsn/binary>> ->
+            cmp(Dep, rm_ws(Vsn), Repo, HexRegistry, State, fun ec_semver:gt/2);
+        <<"<=", Vsn/binary>> ->
+            cmpl(Dep, rm_ws(Vsn), Repo, HexRegistry, State, fun ec_semver:lte/2);
+        <<"<", Vsn/binary>> ->
+            cmpl(Dep, rm_ws(Vsn), Repo, HexRegistry, State, fun ec_semver:lt/2);
+        <<"==", Vsn/binary>> ->
+            {ok, Vsn};
+        Vsn ->
+            {ok, Vsn}
+    end.
+
+rm_ws(<<" ", R/binary>>) ->
+    ec_semver:parse(rm_ws(R));
+rm_ws(R) ->
+    ec_semver:parse(R).
+
+valid_vsn(Vsn) ->
+    %% Regepx from https://github.com/sindresorhus/semver-regex/blob/master/index.js
+    SemVerRegExp = "v?(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)(\\.(0|[1-9][0-9]*))?"
+        "(-[0-9a-z-]+(\\.[0-9a-z-]+)*)?(\\+[0-9a-z-]+(\\.[0-9a-z-]+)*)?",
+    SupportedVersions = "^(>=?|<=?|~>|==)?\\s*" ++ SemVerRegExp ++ "$",
+    re:run(Vsn, SupportedVersions, [unicode]) =/= nomatch.
+
+highest_matching(Dep, Vsn, Repo, HexRegistry, State) ->
+    find_highest_matching_(Dep, Vsn, #{name => Repo}, HexRegistry, State).
+
+cmp(Dep, Vsn, Repo, HexRegistry, State, CmpFun) ->
+    case get_package_versions(Dep, Vsn, Repo, HexRegistry, State) of
+        [] ->
+            none;
+        Vsns ->
+            cmp_(undefined, Vsn, Vsns, CmpFun)
+    end.
+
+cmp_(undefined, MinVsn, [], _CmpFun) ->
+    {ok, MinVsn};
+cmp_(HighestDepVsn, _MinVsn, [], _CmpFun) ->
+    {ok, HighestDepVsn};
+
+cmp_(BestMatch, MinVsn, [Vsn | R], CmpFun) ->
+    case CmpFun(Vsn, MinVsn) of
+        true ->
+            cmp_(Vsn, Vsn, R, CmpFun);
+        false  ->
+            cmp_(BestMatch, MinVsn, R, CmpFun)
+    end.
+
+%% We need to treat this differently since we want a version that is LOWER but
+%% the higest possible one.
+cmpl(Dep, Vsn, Repo, HexRegistry, State, CmpFun) ->
+    case get_package_versions(Dep, Vsn, Repo, HexRegistry, State) of
+        [] ->
+            none;
+        Vsns ->
+            cmpl_(undefined, Vsn, Vsns, CmpFun)
+    end.
+
+cmpl_(undefined, MaxVsn, [], _CmpFun) ->
+    {ok, MaxVsn};
+cmpl_(HighestDepVsn, _MaxVsn, [], _CmpFun) ->
+    {ok, HighestDepVsn};
+
+cmpl_(undefined, MaxVsn, [Vsn | R], CmpFun) ->
+    case CmpFun(Vsn, MaxVsn) of
+        true ->
+            cmpl_(Vsn, MaxVsn, R, CmpFun);
+        false  ->
+            cmpl_(undefined, MaxVsn, R, CmpFun)
+    end;
+
+cmpl_(BestMatch, MaxVsn, [Vsn | R], CmpFun) ->
+    case CmpFun(Vsn, MaxVsn) of
+        true ->
+            case ec_semver:gte(Vsn, BestMatch) of
+                true ->
+                    cmpl_(Vsn, MaxVsn, R, CmpFun);
+                false ->
+                    cmpl_(BestMatch, MaxVsn, R, CmpFun)
+            end;
+        false  ->
+            cmpl_(BestMatch, MaxVsn, R, CmpFun)
+    end.

src/rebar_paths.erl

@@ -0,0 +1,208 @@
+-module(rebar_paths).
+-include("rebar.hrl").
+
+-type target() :: deps | plugins.
+-type targets() :: [target(), ...].
+-export_type([target/0, targets/0]).
+-export([set_paths/2, unset_paths/2]).
+-export([clashing_apps/2]).
+
+-ifdef(TEST).
+-export([misloaded_modules/2]).
+-endif.
+
+-spec set_paths(targets(), rebar_state:t()) -> ok.
+set_paths(UserTargets, State) ->
+    Targets = normalize_targets(UserTargets),
+    GroupPaths = path_groups(Targets, State),
+    Paths = lists:append(lists:reverse([P || {_, P} <- GroupPaths])),
+    code:add_pathsa(Paths),
+    AppGroups = app_groups(Targets, State),
+    purge_and_load(AppGroups, sets:new()),
+    ok.
+
+-spec unset_paths(targets(), rebar_state:t()) -> ok.
+unset_paths(UserTargets, State) ->
+    Targets = normalize_targets(UserTargets),
+    GroupPaths = path_groups(Targets, State),
+    Paths = lists:append([P || {_, P} <- GroupPaths]),
+    [code:del_path(P) || P <- Paths],
+    purge(Paths, code:all_loaded()),
+    ok.
+
+-spec clashing_apps(targets(), rebar_state:t()) -> [{target(), [binary()]}].
+clashing_apps(Targets, State) ->
+    AppGroups = app_groups(Targets, State),
+    AppNames = [{G, sets:from_list(
+                    [rebar_app_info:name(App) || App <- Apps]
+                )} || {G, Apps} <- AppGroups],
+    clashing_app_names(sets:new(), AppNames, []).
+
+%%%%%%%%%%%%%%%
+%%% PRIVATE %%%
+%%%%%%%%%%%%%%%
+
+%% The paths are to be set in the reverse order; i.e. the default
+%% path is always last when possible (minimize cases where a build
+%% tool version clashes with an app's), and put the highest priorities
+%% first.
+-spec normalize_targets(targets()) -> targets().
+normalize_targets(List) ->
+    %% Plan for the eventuality of getting values piped in
+    %% from future versions of rebar3, possibly from plugins and so on,
+    %% which means we'd risk failing kind of violently. We only support
+    %% deps and plugins
+    TmpList = lists:foldl(
+      fun(deps, [deps | _] = Acc) -> Acc;
+         (plugins, [plugins | _] = Acc) -> Acc;
+         (deps, Acc) -> [deps | Acc -- [deps]];
+         (plugins, Acc) -> [plugins | Acc -- [plugins]];
+         (_, Acc) -> Acc
+      end,
+      [],
+      List
+    ),
+    lists:reverse(TmpList).
+
+purge_and_load([], _) ->
+    ok;
+purge_and_load([{_Group, Apps}|Rest], Seen) ->
+    %% We have: a list of all applications in the current priority group,
+    %% a list of all loaded modules with their active path, and a list of
+    %% seen applications.
+    %%
+    %% We do the following:
+    %% 1. identify the apps that have not been solved yet
+    %% 2. find the paths for all apps in the current group
+    %% 3. unload and reload apps that may have changed paths in order
+    %%    to get updated module lists and specs
+    %%    (we ignore started apps and apps that have not run for this)
+    %%    This part turns out to be the bottleneck of this module, so
+    %%    to speed it up, using clash detection proves useful:
+    %%    only reload apps that clashed since others are unlikely to
+    %%    conflict in significant ways
+    %% 4. create a list of modules to check from that app list—only loaded
+    %%    modules make sense to check.
+    %% 5. check the modules to match their currently loaded paths with
+    %%    the path set from the apps in the current group; modules
+    %%    that differ must be purged; others can stay
+
+    %% 1)
+    AppNames = [AppName || App <- Apps,
+                           AppName <- [rebar_app_info:name(App)],
+                           not sets:is_element(AppName, Seen)],
+    GoodApps = [App || AppName <- AppNames,
+                       App <- Apps,
+                       rebar_app_info:name(App) =:= AppName],
+    %% 2)
+    %% (no need for extra_src_dirs since those get put into ebin;
+    %%  also no need for OTP libs; we want to allow overtaking them)
+    GoodAppPaths = [rebar_app_info:ebin_dir(App) || App <- GoodApps],
+    %% 3)
+    [begin
+         AtomApp = binary_to_atom(AppName, utf8),
+         %% blind load/unload won't interrupt an already-running app,
+         %% preventing odd errors, maybe!
+         case application:unload(AtomApp) of
+             ok -> application:load(AtomApp);
+             _ -> ok
+         end
+     end || AppName <- AppNames,
+            %% Shouldn't unload ourselves; rebar runs without ever
+            %% being started and unloading breaks logging!
+            AppName =/= <<"rebar">>],
+
+    %% 4)
+    CandidateMods = lists:append(
+        %% Start by asking the currently loaded app (if loaded)
+        %% since it would be the primary source of conflicting modules
+        [case application:get_key(AppName, modules) of
+             {ok, Mods} ->
+                 Mods;
+             undefined ->
+                 %% if not found, parse the app file on disk, in case
+                 %% the app's modules are used without it being loaded
+                 case rebar_app_info:app_details(App) of
+                     [] -> [];
+                     Details -> proplists:get_value(modules, Details, [])
+                 end
+         end || App <- GoodApps,
+                AppName <- [binary_to_atom(rebar_app_info:name(App), utf8)]]
+    ),
+    ModPaths = [{Mod,Path} || Mod <- CandidateMods,
+                              erlang:function_exported(Mod, module_info, 0),
+                              {file, Path} <- [code:is_loaded(Mod)]],
+
+    %% 5)
+    Mods = misloaded_modules(GoodAppPaths, ModPaths),
+    [purge_mod(Mod) || Mod <- Mods],
+
+    purge_and_load(Rest, sets:union(Seen, sets:from_list(AppNames))).
+
+purge(Paths, ModPaths) ->
+    SortedPaths = lists:sort(Paths),
+    lists:map(fun purge_mod/1,
+              [Mod || {Mod, Path} <- ModPaths,
+                      is_list(Path), % not 'preloaded' or mocked
+                      any_prefix(Path, SortedPaths)]
+             ).
+
+misloaded_modules(GoodAppPaths, ModPaths) ->
+    %% Identify paths that are invalid; i.e. app paths that cover an
+    %% app in the desired group, but are not in the desired group.
+    lists:usort(
+        [Mod || {Mod, Path} <- ModPaths,
+                is_list(Path), % not 'preloaded' or mocked
+                not any_prefix(Path, GoodAppPaths)]
+    ).
+
+any_prefix(Path, Paths) ->
+    lists:any(fun(P) -> lists:prefix(P, Path) end, Paths).
+
+%% assume paths currently set are good; only unload a module so next call
+%% uses the correctly set paths
+purge_mod(Mod) ->
+    code:soft_purge(Mod) andalso code:delete(Mod).
+
+
+%% This is a tricky O(n²) check since we want to
+%% know whether an app clashes with any of the top priority groups.
+%%
+%% For example, let's say we have `[deps, plugins]', then we want
+%% to find the plugins that clash with deps:
+%%
+%% `[{deps, [ClashingPlugins]}, {plugins, []}]'
+%%
+%% In case we'd ever have alternative or additional types, we can
+%% find all clashes from other 'groups'.
+clashing_app_names(_, [], Acc) ->
+    lists:reverse(Acc);
+clashing_app_names(PrevNames, [{G,AppNames} | Rest], Acc) ->
+    CurrentNames = sets:subtract(AppNames, PrevNames),
+    NextNames = sets:subtract(sets:union([A || {_, A} <- Rest]), PrevNames),
+    Clashes = sets:intersection(CurrentNames, NextNames),
+    NewAcc = [{G, sets:to_list(Clashes)} | Acc],
+    clashing_app_names(sets:union(PrevNames, CurrentNames), Rest, NewAcc).
+
+path_groups(Targets, State) ->
+    [{Target, get_paths(Target, State)} || Target <- Targets].
+
+app_groups(Targets, State) ->
+    [{Target, get_apps(Target, State)} || Target <- Targets].
+
+get_paths(deps, State) ->
+    rebar_state:code_paths(State, all_deps);
+get_paths(plugins, State) ->
+    rebar_state:code_paths(State, all_plugin_deps).
+
+get_apps(deps, State) ->
+    %% The code paths for deps also include the top level apps
+    %% and the extras, which we don't have here; we have to
+    %% add the apps by hand
+    case rebar_state:project_apps(State) of
+        undefined -> [];
+        List -> List
+    end ++
+    rebar_state:all_deps(State);
+get_apps(plugins, State) ->
+    rebar_state:all_plugin_deps(State).

src/rebar_pkg_resource.erl

@@ -2,42 +2,51 @@
 %% ex: ts=4 sw=4 et
 -module(rebar_pkg_resource).
 
--behaviour(rebar_resource).
+-behaviour(rebar_resource_v2).
 
--export([lock/2
-        ,download/3
-        ,download/4
-        ,needs_update/2
-        ,make_vsn/1]).
+-export([init/2,
+         lock/2,
+         download/4,
+         download/5,
+         needs_update/2,
+         make_vsn/2,
+         format_error/1]).
 
--export([request/2
-        ,etag/1
-        ,ssl_opts/1]).
-
-%% Exported for ct
+-ifdef(TEST).
+%% exported for test purposes
 -export([store_etag_in_cache/2]).
+-endif.
 
 -include("rebar.hrl").
--include_lib("public_key/include/OTP-PUB-KEY.hrl").
-
--type cached_result()   :: {'bad_checksum',string()} |
-                           {'bad_registry_checksum',string()} |
-                           {'failed_extract',string()} |
-                           {'ok','true'} |
-                           {'unexpected_hash',string(),_,binary()}.
+-include_lib("providers/include/providers.hrl").
 
--type download_result() :: {bad_download, binary() | string()} |
-                           {fetch_fail, _, _} | cached_result().
+-type package() :: {pkg, binary(), binary(), binary(), rebar_hex_repos:repo()}.
 
 %%==============================================================================
 %% Public API
 %%==============================================================================
--spec lock(AppDir, Source) -> Res when
-      AppDir :: file:name(),
-      Source :: tuple(),
-      Res :: {atom(), string(), any()}.
-lock(_AppDir, Source) ->
-    Source.
+
+-spec init(atom(), rebar_state:t()) -> {ok, rebar_resource_v2:resource()}.
+init(Type, State) ->
+    {ok, Vsn} = application:get_key(rebar, vsn),
+    BaseConfig = #{http_adapter => hex_http_httpc,
+                   http_user_agent_fragment =>
+                       <<"(rebar3/", (list_to_binary(Vsn))/binary, ") (httpc)">>,
+                   http_adapter_config => #{profile => rebar}},
+    Repos = rebar_hex_repos:from_state(BaseConfig, State),
+    Resource = rebar_resource_v2:new(Type, ?MODULE, #{repos => Repos,
+                                                      base_config => BaseConfig}),
+    {ok, Resource}.
+
+
+
+-spec lock(AppInfo, ResourceState) -> Res when
+      AppInfo :: rebar_app_info:t(),
+      ResourceState :: rebar_resource_v2:resource_state(),
+      Res :: {atom(), string(), any(), binary()}.
+lock(AppInfo, _) ->
+    {pkg, Name, Vsn, Hash, _RepoConfig} = rebar_app_info:source(AppInfo),
+    {pkg, Name, Vsn, Hash}.
 
 %%------------------------------------------------------------------------------
 %% @doc
@@ -45,13 +54,13 @@ lock(_AppDir, Source) ->
 %% version.
 %% @end
 %%------------------------------------------------------------------------------
--spec needs_update(Dir, Pkg) -> Res when
-      Dir :: file:name(),
-      Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()},
+-spec needs_update(AppInfo, ResourceState) -> Res when
+      AppInfo :: rebar_app_info:t(),
+      ResourceState :: rebar_resource_v2:resource_state(),
       Res :: boolean().
-needs_update(Dir, {pkg, _Name, Vsn, _Hash}) ->
-    [AppInfo] = rebar_app_discover:find_apps([Dir], all),
-    case rebar_app_info:original_vsn(AppInfo) =:= rebar_utils:to_list(Vsn) of
+needs_update(AppInfo, _) ->
+    {pkg, _Name, Vsn, _Hash, _} = rebar_app_info:source(AppInfo),
+    case rebar_utils:to_binary(rebar_app_info:original_vsn(AppInfo)) =:= rebar_utils:to_binary(Vsn) of
         true ->
             false;
         false ->
@@ -63,13 +72,19 @@ needs_update(Dir, {pkg, _Name, Vsn, _Hash}) ->
 %% Download the given pkg.
 %% @end
 %%------------------------------------------------------------------------------
--spec download(TmpDir, Pkg, State) -> Res when
+-spec download(TmpDir, AppInfo, State, ResourceState) -> Res when
       TmpDir :: file:name(),
-      Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()},
+      AppInfo :: rebar_app_info:t(),
+      ResourceState :: rebar_resource_v2:resource_state(),
       State :: rebar_state:t(),
-      Res :: {'error',_} | {'ok',_} | {'tarball',binary() | string()}.
-download(TmpDir, Pkg, State) ->
-    download(TmpDir, Pkg, State, true).
+      Res :: ok | {error,_}.
+download(TmpDir, AppInfo, State, ResourceState) ->
+    case download(TmpDir, rebar_app_info:source(AppInfo), State, ResourceState, true) of
+        ok ->
+            ok;
+        Error ->
+            {error, Error}
+    end.
 
 %%------------------------------------------------------------------------------
 %% @doc
@@ -78,26 +93,28 @@ download(TmpDir, Pkg, State) ->
 %% is different.
 %% @end
 %%------------------------------------------------------------------------------
--spec download(TmpDir, Pkg, State, UpdateETag) -> Res when
+-spec download(TmpDir, Pkg, State, ResourceState, UpdateETag) -> Res when
       TmpDir :: file:name(),
-      Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()},
+      Pkg :: package(),
       State :: rebar_state:t(),
+      ResourceState:: rebar_resource_v2:resource_state(),
       UpdateETag :: boolean(),
-      Res :: download_result().
-download(TmpDir, Pkg={pkg, Name, Vsn, _Hash}, State, UpdateETag) ->
-    CDN = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN),
-    {ok, PackageDir} = rebar_packages:package_dir(State),
+      Res :: ok | {error,_} | {unexpected_hash, string(), integer(), integer()} |
+             {fetch_fail, binary(), binary()}.
+download(TmpDir, Pkg={pkg, Name, Vsn, _Hash, Repo}, State, _ResourceState, UpdateETag) ->
+    {ok, PackageDir} = rebar_packages:package_dir(Repo, State),
     Package = binary_to_list(<<Name/binary, "-", Vsn/binary, ".tar">>),
     ETagFile = binary_to_list(<<Name/binary, "-", Vsn/binary, ".etag">>),
     CachePath = filename:join(PackageDir, Package),
     ETagPath = filename:join(PackageDir, ETagFile),
-    case rebar_utils:url_append_path(CDN, filename:join(?REMOTE_PACKAGE_DIR,
-                                                        Package)) of
-        {ok, Url} ->
-            cached_download(TmpDir, CachePath, Pkg, Url, etag(ETagPath), State,
-                            ETagPath, UpdateETag);
-        _ ->
-            {fetch_fail, Name, Vsn}
+    case cached_download(TmpDir, CachePath, Pkg, etag(CachePath, ETagPath), ETagPath, UpdateETag) of
+        {bad_registry_checksum, Expected, Found} ->
+            %% checksum comparison failed. in case this is from a modified cached package
+            %% overwrite the etag if it exists so it is not relied on again
+            store_etag_in_cache(ETagPath, <<>>),
+            ?PRV_ERROR({bad_registry_checksum, Name, Vsn, Expected, Found});
+        Result ->
+            Result
     end.
 
 %%------------------------------------------------------------------------------
@@ -106,12 +123,19 @@ download(TmpDir, Pkg={pkg, Name, Vsn, _Hash}, State, UpdateETag) ->
 %% Returns {error, string()} as this operation is not supported for pkg sources.
 %% @end
 %%------------------------------------------------------------------------------
--spec make_vsn(Vsn) -> Res when
-      Vsn :: any(),
-      Res :: {'error',[1..255,...]}.
-make_vsn(_) ->
+-spec make_vsn(AppInfo, ResourceState) -> Res when
+      AppInfo :: rebar_app_info:t(),
+      ResourceState :: rebar_resource_v2:resource_state(),
+      Res :: {'error', string()}.
+make_vsn(_, _) ->
     {error, "Replacing version of type pkg not supported."}.
 
+format_error({bad_registry_checksum, Name, Vsn, Expected, Found}) ->
+    io_lib:format("The checksum for package at ~ts-~ts (~ts) does not match the "
+                  "checksum expected from the registry (~ts). "
+                  "Run `rebar3 do unlock ~ts, update` and then try again.",
+                  [Name, Vsn, Found, Expected, Name]).
+
 %%------------------------------------------------------------------------------
 %% @doc
 %% Download the pkg belonging to the given address. If the etag of the pkg
@@ -120,29 +144,24 @@ make_vsn(_) ->
 %% {ok, Contents, NewEtag}, otherwise if some error occured return error.
 %% @end
 %%------------------------------------------------------------------------------
--spec request(Url, ETag) -> Res when
-      Url :: string(),
-      ETag :: false | string(),
-      Res :: 'error' | {ok, cached} | {ok, any(), string()}.
-request(Url, ETag) ->
-    HttpOptions = [{ssl, ssl_opts(Url)},
-                   {relaxed, true} | rebar_utils:get_proxy_auth()],
-    case httpc:request(get, {Url, [{"if-none-match", "\"" ++ ETag ++ "\""}
-                                   || ETag =/= false] ++
-                             [{"User-Agent", rebar_utils:user_agent()}]},
-                       HttpOptions, [{body_format, binary}], rebar) of
-        {ok, {{_Version, 200, _Reason}, Headers, Body}} ->
-            ?DEBUG("Successfully downloaded ~ts", [Url]),
-            {"etag", ETag1} = lists:keyfind("etag", 1, Headers),
-            {ok, Body, rebar_string:trim(ETag1, both, [$"])};
-        {ok, {{_Version, 304, _Reason}, _Headers, _Body}} ->
-            ?DEBUG("Cached copy of ~ts still valid", [Url]),
+-spec request(rebar_hex_repos:repo(), binary(), binary(), false | binary())
+             -> {ok, cached} | {ok, binary(), binary()} | error.
+request(Config, Name, Version, ETag) ->
+    Config1 = Config#{http_etag => ETag},
+    try hex_repo:get_tarball(Config1, Name, Version) of
+        {ok, {200, #{<<"etag">> := ETag1}, Tarball}} ->
+            {ok, Tarball, ETag1};
+        {ok, {304, _Headers, _}} ->
             {ok, cached};
-        {ok, {{_Version, Code, _Reason}, _Headers, _Body}} ->
-            ?DEBUG("Request to ~p failed: status code ~p", [Url, Code]),
+        {ok, {Code, _Headers, _Body}} ->
+            ?DEBUG("Request for package ~s-~s failed: status code ~p", [Name, Version, Code]),
             error;
         {error, Reason} ->
-            ?DEBUG("Request to ~p failed: ~p", [Url, Reason]),
+            ?DEBUG("Request for package ~s-~s failed: ~p", [Name, Version, Reason]),
+            error
+    catch
+        _:Exception ->
+            ?DEBUG("hex_repo:get_tarball failed: ~p", [Exception]),
             error
     end.
 
@@ -153,32 +172,23 @@ request(Url, ETag) ->
 %% returned from the hexpm server. The name is package-vsn.etag.
 %% @end
 %%------------------------------------------------------------------------------
--spec etag(Path) -> Res when
-      Path :: file:name(),
-      Res :: false | string().
-etag(Path) ->
-    case file:read_file(Path) of
+-spec etag(PackagePath, ETagPath) -> Res when
+      PackagePath :: file:name(),
+      ETagPath :: file:name(),
+      Res :: binary().
+etag(PackagePath, ETagPath) ->
+    case file:read_file(ETagPath) of
         {ok, Bin} ->
-            binary_to_list(Bin);
+            %% just in case a user deleted a cached package but not its etag
+            %% verify the package is also there, and if not, ignore the etag
+            case filelib:is_file(PackagePath) of
+                true ->
+                    Bin;
+                false ->
+                    <<>>
+            end;
         {error, _} ->
-            false
-    end.
-
-%%------------------------------------------------------------------------------
-%% @doc
-%% Return the SSL options adequate for the project based on
-%% its configuration, including for validation of certs.
-%% @end
-%%------------------------------------------------------------------------------
--spec ssl_opts(Url) -> Res when
-      Url :: string(),
-      Res :: proplists:proplist().
-ssl_opts(Url) ->
-    case get_ssl_config() of
-        ssl_verify_enabled ->
-            ssl_opts(ssl_verify_enabled, Url);
-        ssl_verify_disabled ->
-            [{verify, verify_none}]
+            <<>>
     end.
 
 %%------------------------------------------------------------------------------
@@ -188,7 +198,7 @@ ssl_opts(Url) ->
 %%------------------------------------------------------------------------------
 -spec store_etag_in_cache(File, ETag) -> Res when
       File :: file:name(),
-      ETag :: string(),
+      ETag :: binary(),
       Res :: ok.
 store_etag_in_cache(Path, ETag) ->
     _ = file:write_file(Path, ETag).
@@ -196,223 +206,74 @@ store_etag_in_cache(Path, ETag) ->
 %%%=============================================================================
 %%% Private functions
 %%%=============================================================================
--spec cached_download(TmpDir, CachePath, Pkg, Url, ETag, State, ETagPath,
-                      UpdateETag) -> Res when
+-spec cached_download(TmpDir, CachePath, Pkg, ETag, ETagPath, UpdateETag) -> Res when
       TmpDir :: file:name(),
       CachePath :: file:name(),
-      Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()},
-      Url :: string(),
-      ETag :: false | string(),
-      State :: rebar_state:t(),
+      Pkg :: package(),
+      ETag :: binary(),
       ETagPath :: file:name(),
       UpdateETag :: boolean(),
-      Res :: download_result().
-cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn, _Hash}, Url, ETag,
-                State, ETagPath, UpdateETag) ->
-    case request(Url, ETag) of
+      Res :: ok | {unexpected_hash, integer(), integer()} | {fetch_fail, binary(), binary()}.
+cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn, _Hash, RepoConfig}, ETag,
+                ETagPath, UpdateETag) ->
+    case request(RepoConfig, Name, Vsn, ETag) of
         {ok, cached} ->
             ?INFO("Version cached at ~ts is up to date, reusing it", [CachePath]),
-            serve_from_cache(TmpDir, CachePath, Pkg, State);
+            serve_from_cache(TmpDir, CachePath, Pkg);
         {ok, Body, NewETag} ->
             ?INFO("Downloaded package, caching at ~ts", [CachePath]),
             maybe_store_etag_in_cache(UpdateETag, ETagPath, NewETag),
-            serve_from_download(TmpDir, CachePath, Pkg, NewETag, Body, State,
-                                ETagPath);
-        error when ETag =/= false ->
+            serve_from_download(TmpDir, CachePath, Pkg, Body);
+        error when ETag =/= <<>> ->
             store_etag_in_cache(ETagPath, ETag),
             ?INFO("Download error, using cached file at ~ts", [CachePath]),
-            serve_from_cache(TmpDir, CachePath, Pkg, State);
+            serve_from_cache(TmpDir, CachePath, Pkg);
         error ->
             {fetch_fail, Name, Vsn}
     end.
 
--spec serve_from_cache(TmpDir, CachePath, Pkg, State) -> Res when
+-spec serve_from_cache(TmpDir, CachePath, Pkg) -> Res when
       TmpDir :: file:name(),
       CachePath :: file:name(),
-      Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()},
-      State :: rebar_state:t(),
-      Res :: cached_result().
-serve_from_cache(TmpDir, CachePath, Pkg, State) ->
-    {Files, Contents, Version, Meta} = extract(TmpDir, CachePath),
-    case checksums(Pkg, Files, Contents, Version, Meta, State) of
-        {Chk, Chk, Chk, Chk} ->
-            ok = erl_tar:extract({binary, Contents}, [{cwd, TmpDir}, compressed]),
-            {ok, true};
-        {_Hash, Chk, Chk, Chk} ->
-            ?DEBUG("Expected hash ~p does not match checksums ~p", [_Hash, Chk]),
-            {unexpected_hash, CachePath, _Hash, Chk};
-        {Chk, _Bin, Chk, Chk} ->
-            ?DEBUG("Checksums: registry: ~p, pkg: ~p", [Chk, _Bin]),
-            {failed_extract, CachePath};
-        {Chk, Chk, _Reg, Chk} ->
-            ?DEBUG("Checksums: registry: ~p, pkg: ~p", [_Reg, Chk]),
-            {bad_registry_checksum, CachePath};
-        {_Hash, _Bin, _Reg, _Tar} ->
-            ?DEBUG("Checksums: expected: ~p, registry: ~p, pkg: ~p, meta: ~p",
-                   [_Hash, _Reg, _Bin, _Tar]),
-            {bad_checksum, CachePath}
+      Pkg :: package(),
+      Res :: ok | {error,_} | {bad_registry_checksum, integer(), integer()}.
+serve_from_cache(TmpDir, CachePath, Pkg) ->
+    {ok, Binary} = file:read_file(CachePath),
+    serve_from_memory(TmpDir, Binary, Pkg).
+
+-spec serve_from_memory(TmpDir, Tarball, Package) -> Res when
+      TmpDir :: file:name(),
+      Tarball :: binary(),
+      Package :: package(),
+      Res :: ok | {error,_} | {bad_registry_checksum, integer(), integer()}.
+serve_from_memory(TmpDir, Binary, {pkg, _Name, _Vsn, Hash, _RepoConfig}) ->;
+    RegistryChecksum = list_to_integer(binary_to_list(Hash), 16),
+    case hex_tarball:unpack(Binary, TmpDir) of
+        {ok, #{checksum := <<Checksum:256/big-unsigned>>}} when RegistryChecksum =/= Checksum ->;
+            ?DEBUG("Expected hash ~64.16.0B does not match checksum of fetched package ~64.16.0B",
+                   [RegistryChecksum, Checksum]),
+            {bad_registry_checksum, RegistryChecksum, Checksum};
+        {ok, #{checksum := <<RegistryChecksum:256/big-unsigned>>}} ->
+            ok;
+        {error, Reason} ->
+            {error, {hex_tarball, Reason}}
     end.
 
--spec serve_from_download(TmpDir, CachePath, Package, ETag, Binary, State,
-                          ETagPath) -> Res when
+-spec serve_from_download(TmpDir, CachePath, Package, Binary) -> Res when
       TmpDir :: file:name(),
       CachePath :: file:name(),
-      Package :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()},
-      ETag :: string(),
+      Package :: package(),
       Binary :: binary(),
-      State :: rebar_state:t(),
-      ETagPath :: file:name(),
-      Res :: download_result().
-serve_from_download(TmpDir, CachePath, Package, ETag, Binary, State, ETagPath) ->
+      Res :: ok | {error,_}.
+serve_from_download(TmpDir, CachePath, Package, Binary) ->
     ?DEBUG("Writing ~p to cache at ~ts", [Package, CachePath]),
     file:write_file(CachePath, Binary),
-    case etag(ETagPath) of
-        ETag ->
-            serve_from_cache(TmpDir, CachePath, Package, State);
-        FileETag ->
-            ?DEBUG("Downloaded file ~ts ETag ~ts doesn't match returned ETag ~ts",
-                   [CachePath, ETag, FileETag]),
-            {bad_download, CachePath}
-    end.
-
--spec extract(TmpDir, CachePath) -> Res when
-      TmpDir :: file:name(),
-      CachePath :: file:name(),
-      Res :: {Files, Contents, Version, Meta},
-      Files :: list({file:name(), binary()}),
-      Contents :: binary(),
-      Version :: binary(),
-      Meta :: binary().
-extract(TmpDir, CachePath) ->
-    ec_file:mkdir_p(TmpDir),
-    {ok, Files} = erl_tar:extract(CachePath, [memory]),
-    {"contents.tar.gz", Contents} = lists:keyfind("contents.tar.gz", 1, Files),
-    {"VERSION", Version} = lists:keyfind("VERSION", 1, Files),
-    {"metadata.config", Meta} = lists:keyfind("metadata.config", 1, Files),
-    {Files, Contents, Version, Meta}.
-
--spec checksums(Pkg, Files, Contents, Version, Meta, State) -> Res when
-      Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()},
-      Files :: list({file:name(), binary()}),
-      Contents :: binary(),
-      Version :: binary(),
-      Meta :: binary(),
-      State :: rebar_state:t(),
-      Res :: {Hash, BinChecksum, RegistryChecksum, TarChecksum},
-      Hash :: binary(),
-      BinChecksum :: binary(),
-      RegistryChecksum :: any(),
-      TarChecksum :: binary().
-checksums(Pkg={pkg, _Name, _Vsn, Hash}, Files, Contents, Version, Meta, State) ->
-    Blob = <<Version/binary, Meta/binary, Contents/binary>>,
-    <<X:256/big-unsigned>> = crypto:hash(sha256, Blob),
-    BinChecksum = list_to_binary(
-                    rebar_string:uppercase(
-                      lists:flatten(io_lib:format("~64.16.0b", [X])))),
-    RegistryChecksum = rebar_packages:registry_checksum(Pkg, State),
-    {"CHECKSUM", TarChecksum} = lists:keyfind("CHECKSUM", 1, Files),
-    {Hash, BinChecksum, RegistryChecksum, TarChecksum}.
-
-%%------------------------------------------------------------------------------
-%% @doc
-%% Return the SSL options adequate for the project based on
-%% its configuration, including for validation of certs.
-%% @end
-%%------------------------------------------------------------------------------
--spec ssl_opts(Enabled, Url) -> Res when
-      Enabled :: atom(),
-      Url :: string(),
-      Res :: proplists:proplist().
-ssl_opts(ssl_verify_enabled, Url) ->
-    case check_ssl_version() of
-        true ->
-            {ok, {_, _, Hostname, _, _, _}} =
-                http_uri:parse(rebar_utils:to_list(Url)),
-            VerifyFun = {fun ssl_verify_hostname:verify_fun/3,
-                         [{check_hostname, Hostname}]},
-            CACerts = certifi:cacerts(),
-            [{verify, verify_peer}, {depth, 2}, {cacerts, CACerts},
-             {partial_chain, fun partial_chain/1}, {verify_fun, VerifyFun}];
-        false ->
-            ?WARN("Insecure HTTPS request (peer verification disabled), "
-                  "please update to OTP 17.4 or later", []),
-            [{verify, verify_none}]
-    end.
-
--spec partial_chain(Certs) -> Res when
-      Certs :: list(any()),
-      Res :: unknown_ca | {trusted_ca, any()}.
-partial_chain(Certs) ->
-    Certs1 = [{Cert, public_key:pkix_decode_cert(Cert, otp)} || Cert <- Certs],
-    CACerts = certifi:cacerts(),
-    CACerts1 = [public_key:pkix_decode_cert(Cert, otp) || Cert <- CACerts],
-    case ec_lists:find(fun({_, Cert}) ->
-                               check_cert(CACerts1, Cert)
-                       end, Certs1) of
-        {ok, Trusted} ->
-            {trusted_ca, element(1, Trusted)};
-        _ ->
-            unknown_ca
-    end.
-
--spec extract_public_key_info(Cert) -> Res when
-      Cert :: #'OTPCertificate'{tbsCertificate::#'OTPTBSCertificate'{}},
-      Res :: any().
-extract_public_key_info(Cert) ->
-    ((Cert#'OTPCertificate'.tbsCertificate)#'OTPTBSCertificate'.subjectPublicKeyInfo).
-
--spec check_cert(CACerts, Cert) -> Res when
-      CACerts :: list(any()),
-      Cert :: any(),
-      Res :: boolean().
-check_cert(CACerts, Cert) ->
-    lists:any(fun(CACert) ->
-                      extract_public_key_info(CACert) == extract_public_key_info(Cert)
-              end, CACerts).
-
--spec check_ssl_version() ->
-    boolean().
-check_ssl_version() ->
-    case application:get_key(ssl, vsn) of
-        {ok, Vsn} ->
-            parse_vsn(Vsn) >= {5, 3, 6};
-        _ ->
-            false
-    end.
-
--spec get_ssl_config() ->
-      ssl_verify_disabled | ssl_verify_enabled.
-get_ssl_config() ->
-    GlobalConfigFile = rebar_dir:global_config(),
-    Config = rebar_config:consult_file(GlobalConfigFile),
-    case proplists:get_value(ssl_verify, Config, []) of
-        false ->
-            ssl_verify_disabled;
-        _ ->
-            ssl_verify_enabled
-    end.
-
--spec parse_vsn(Vsn) -> Res when
-      Vsn :: string(),
-      Res :: {integer(), integer(), integer()}.
-parse_vsn(Vsn) ->
-    version_pad(rebar_string:lexemes(Vsn, ".-")).
-
--spec version_pad(list(nonempty_string())) -> Res when
-      Res :: {integer(), integer(), integer()}.
-version_pad([Major]) ->
-    {list_to_integer(Major), 0, 0};
-version_pad([Major, Minor]) ->
-    {list_to_integer(Major), list_to_integer(Minor), 0};
-version_pad([Major, Minor, Patch]) ->
-    {list_to_integer(Major), list_to_integer(Minor), list_to_integer(Patch)};
-version_pad([Major, Minor, Patch | _]) ->
-    {list_to_integer(Major), list_to_integer(Minor), list_to_integer(Patch)}.
+    serve_from_memory(TmpDir, Binary, Package).
 
 -spec maybe_store_etag_in_cache(UpdateETag, Path, ETag) -> Res when
       UpdateETag :: boolean(),
       Path :: file:name(),
-      ETag :: string(),
+      ETag :: binary(),
       Res :: ok.
 maybe_store_etag_in_cache(false = _UpdateETag, _Path, _ETag) ->
     ok;

src/rebar_plugins.erl

@@ -39,13 +39,18 @@ project_apps_install(State) ->
     Profiles = rebar_state:current_profiles(State),
     ProjectApps = rebar_state:project_apps(State),
     lists:foldl(fun(Profile, StateAcc) ->
-                        Plugins = rebar_state:get(State, {plugins, Profile}, []),
-                        StateAcc1 = handle_plugins(Profile, Plugins, StateAcc),
+                        StateAcc1 = case Profile of
+                                        default ->
+                                            %% default profile top level plugins
+                                            %% are installed in run_aux
+                                            StateAcc;
+                                        _ ->
+                                            Plugins = rebar_state:get(State, {plugins, Profile}, []),
+                                            handle_plugins(Profile, Plugins, StateAcc)
+                                    end,
 
                         lists:foldl(fun(AppInfo, StateAcc2) ->
-                                            C = rebar_config:consult(rebar_app_info:dir(AppInfo)),
-                                            AppInfo0 = rebar_app_info:update_opts(AppInfo, rebar_app_info:opts(AppInfo), C),
-                                            Plugins2 = rebar_app_info:get(AppInfo0, {plugins, Profile}, []),
+                                            Plugins2 = rebar_app_info:get(AppInfo, {plugins, Profile}, []),
                                             handle_plugins(Profile, Plugins2, StateAcc2)
                                     end, StateAcc1, ProjectApps)
                 end, State, Profiles).
@@ -62,12 +67,25 @@ install(State, AppInfo) ->
 
     State2 = lists:foldl(fun(Profile, StateAcc) ->
                              Plugins = rebar_app_info:get(AppInfo, {plugins, Profile}, []),
-                             handle_plugins(Profile, Plugins, StateAcc)
+                             Plugins1 = filter_existing_plugins(Plugins, StateAcc),
+                             handle_plugins(Profile, Plugins1, StateAcc)
                          end, State1, Profiles),
 
     %% Reset the overrides after processing the dep
     rebar_state:set(State2, overrides, StateOverrides).
 
+filter_existing_plugins(Plugins, State) ->
+    PluginNames = lists:zip(Plugins, rebar_state:deps_names(Plugins)),
+    AllPlugins = rebar_state:all_plugin_deps(State),
+    lists:filtermap(fun({Plugin, PluginName}) ->
+                            case rebar_app_utils:find(PluginName, AllPlugins) of
+                                {ok, _} ->
+                                    false;
+                                _ ->
+                                    {true, Plugin}
+                            end
+                    end, PluginNames).
+
 handle_plugins(Profile, Plugins, State) ->
     handle_plugins(Profile, Plugins, State, false).
 
@@ -76,7 +94,6 @@ handle_plugins(Profile, Plugins, State, Upgrade) ->
     Locks = rebar_state:lock(State),
     DepsDir = rebar_state:get(State, deps_dir, ?DEFAULT_DEPS_DIR),
     State1 = rebar_state:set(State, deps_dir, ?DEFAULT_PLUGINS_DIR),
-
     %% Install each plugin individually so if one fails to install it doesn't effect the others
     {_PluginProviders, State2} =
         lists:foldl(fun(Plugin, {PluginAcc, StateAcc}) ->
@@ -105,12 +122,10 @@ handle_plugin(Profile, Plugin, State, Upgrade) ->
         %% Add newly built deps and plugin to code path
         State3 = rebar_state:update_all_plugin_deps(State2, Apps),
         NewCodePaths = [rebar_app_info:ebin_dir(A) || A <- ToBuild],
-        AllPluginEbins = filelib:wildcard(filename:join([rebar_dir:plugins_dir(State), "*", "ebin"])),
-        CodePaths = PreBuiltPaths++(AllPluginEbins--ToBuild),
-        code:add_pathsa(NewCodePaths++CodePaths),
 
         %% Store plugin code paths so we can remove them when compiling project apps
         State4 = rebar_state:update_code_paths(State3, all_plugin_deps, PreBuiltPaths++NewCodePaths),
+        rebar_paths:set_paths([plugins], State4),
 
         {plugin_providers(Plugin), State4}
     catch
@@ -122,8 +137,6 @@ handle_plugin(Profile, Plugin, State, Upgrade) ->
 
 build_plugin(AppInfo, Apps, State) ->
     Providers = rebar_state:providers(State),
-    %Providers1 = rebar_state:providers(rebar_app_info:state(AppInfo)),
-    %rebar_app_info:state_or_new(State, AppInfo)
     S = rebar_state:all_deps(State, Apps),
     S1 = rebar_state:set(S, deps_dir, ?DEFAULT_PLUGINS_DIR),
     rebar_prv_compile:compile(S1, Providers, AppInfo).

src/rebar_prv_clean.erl

@@ -67,11 +67,12 @@ format_error(Reason) ->
 %% ===================================================================
 
 clean_apps(State, Providers, Apps) ->
+    Compilers = rebar_state:compilers(State),
     [begin
          ?INFO("Cleaning out ~ts...", [rebar_app_info:name(AppInfo)]),
          AppDir = rebar_app_info:dir(AppInfo),
          AppInfo1 = rebar_hooks:run_all_hooks(AppDir, pre, ?PROVIDER, Providers, AppInfo, State),
-         rebar_erlc_compiler:clean(AppInfo1),
+         rebar_compiler:clean(Compilers, AppInfo1),
          rebar_hooks:run_all_hooks(AppDir, post, ?PROVIDER, Providers, AppInfo1, State)
      end || AppInfo <- Apps].
 

src/rebar_prv_common_test.erl

@@ -58,7 +58,7 @@ do(State) ->
 
 do(State, Tests) ->
     ?INFO("Running Common Test suites...", []),
-    rebar_utils:update_code(rebar_state:code_paths(State, all_deps), [soft_purge]),
+    rebar_paths:set_paths([deps, plugins], State),
 
     %% Run ct provider prehooks
     Providers = rebar_state:providers(State),
@@ -73,14 +73,14 @@ do(State, Tests) ->
                 ok    ->
                     %% Run ct provider post hooks for all project apps and top level project hooks
                     rebar_hooks:run_project_and_app_hooks(Cwd, post, ?PROVIDER, Providers, State),
-                    rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)),
+                    rebar_paths:set_paths([plugins, deps], State),
                     {ok, State};
                 Error ->
-                    rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)),
+                    rebar_paths:set_paths([plugins, deps], State),
                     Error
             end;
         Error ->
-            rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)),
+            rebar_paths:set_paths([plugins, deps], State),
             Error
     end.
 
@@ -250,11 +250,9 @@ select_tests(State, ProjectApps, CmdOpts, CfgOpts) ->
                             end, SysConfigs),
     %% NB: load the applications (from user directories too) to support OTP < 17
     %% to our best ability.
-    OldPath = code:get_path(),
-    code:add_pathsa(rebar_state:code_paths(State, all_deps)),
+    rebar_paths:set_paths([deps, plugins], State),
     [application:load(Application) || Config <- Configs, {Application, _} <- Config],
     rebar_utils:reread_config(Configs),
-    code:set_path(OldPath),
 
     Opts = merge_opts(CmdOpts,CfgOpts),
     discover_tests(State, ProjectApps, Opts).

src/rebar_prv_compile.erl

@@ -30,22 +30,37 @@ init(State) ->
                                                                {example, "rebar3 compile"},
                                                                {short_desc, "Compile apps .app.src and .erl files."},
                                                                {desc, "Compile apps .app.src and .erl files."},
-                                                               {opts, []}])),
+                                                               {opts, [{deps_only, $d, "deps_only", undefined,
+                                                                        "Only compile dependencies, no project apps will be built."}]}])),
     {ok, State1}.
 
 -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}.
 do(State) ->
-    DepsPaths = rebar_state:code_paths(State, all_deps),
-    PluginDepsPaths = rebar_state:code_paths(State, all_plugin_deps),
-    rebar_utils:remove_from_code_path(PluginDepsPaths),
-    code:add_pathsa(DepsPaths),
+    IsDepsOnly = is_deps_only(State),
+    rebar_paths:set_paths([deps], State),
 
-    ProjectApps = rebar_state:project_apps(State),
     Providers = rebar_state:providers(State),
     Deps = rebar_state:deps_to_build(State),
-    Cwd = rebar_state:dir(State),
-
     copy_and_build_apps(State, Providers, Deps),
+
+    State1 = case IsDepsOnly of
+                 true ->
+                     State;
+                 false ->
+                     handle_project_apps(Providers, State)
+             end,
+
+    rebar_paths:set_paths([plugins], State1),
+
+    {ok, State1}.
+
+is_deps_only(State) ->
+    {Args, _} = rebar_state:command_parsed_args(State),
+    proplists:get_value(deps_only, Args, false).
+
+handle_project_apps(Providers, State) ->
+    Cwd = rebar_state:dir(State),
+    ProjectApps = rebar_state:project_apps(State),
     {ok, ProjectApps1} = rebar_digraph:compile_order(ProjectApps),
 
     %% Run top level hooks *before* project apps compiled but *after* deps are
@@ -57,7 +72,7 @@ do(State) ->
     %% projects with structures like /apps/foo,/apps/bar,/test
     build_extra_dirs(State, ProjectApps2),
 
-    State3 = update_code_paths(State2, ProjectApps2, DepsPaths),
+    State3 = update_code_paths(State2, ProjectApps2),
 
     rebar_hooks:run_all_hooks(Cwd, post, ?PROVIDER, Providers, State2),
     case rebar_state:has_all_artifacts(State3) of
@@ -66,14 +81,19 @@ do(State) ->
         true ->
             true
     end,
-    rebar_utils:cleanup_code_path(rebar_state:code_paths(State3, default)
-                                 ++ rebar_state:code_paths(State, all_plugin_deps)),
 
-    {ok, State3}.
+    State3.
+
 
 -spec format_error(any()) -> iolist().
 format_error({missing_artifact, File}) ->
     io_lib:format("Missing artifact ~ts", [File]);
+format_error({bad_project_builder, Name, Type, Module}) ->
+    io_lib:format("Error building application ~s:~n     Required project builder ~s function "
+                  "~s:build/1 not found", [Name, Type, Module]);
+format_error({unknown_project_type, Name, Type}) ->
+    io_lib:format("Error building application ~s:~n     "
+                  "No project builder is configured for type ~s", [Name, Type]);
 format_error(Reason) ->
     io_lib:format("~p", [Reason]).
 
@@ -95,7 +115,7 @@ copy_and_build_project_apps(State, Providers, Apps) ->
                    rebar_app_info:dir(AppInfo),
                    rebar_app_info:out_dir(AppInfo))
      || AppInfo <- Apps],
-    code:add_pathsa([rebar_app_info:out_dir(AppInfo) || AppInfo <- Apps]),
+    code:add_pathsa([rebar_app_info:ebin_dir(AppInfo) || AppInfo <- Apps]),
     [compile(State, Providers, AppInfo) || AppInfo <- Apps].
 
 
@@ -117,10 +137,19 @@ build_extra_dir(State, Dir) ->
         true ->
             BaseDir = filename:join([rebar_dir:base_dir(State), "extras"]),
             OutDir = filename:join([BaseDir, Dir]),
-            filelib:ensure_dir(filename:join([OutDir, "dummy.beam"])),
+            rebar_file_utils:ensure_dir(OutDir),
             copy(rebar_state:dir(State), BaseDir, Dir),
-            rebar_erlc_compiler:compile_dir(State, BaseDir, OutDir);
-        false -> ok
+
+            Compilers = rebar_state:compilers(State),
+            FakeApp = rebar_app_info:new(),
+            FakeApp1 = rebar_app_info:out_dir(FakeApp, BaseDir),
+            FakeApp2 = rebar_app_info:ebin_dir(FakeApp1, OutDir),
+            Opts = rebar_state:opts(State),
+            FakeApp3 = rebar_app_info:opts(FakeApp2, Opts),
+            FakeApp4 = rebar_app_info:set(FakeApp3, src_dirs, [OutDir]),
+            rebar_compiler:compile_all(Compilers, FakeApp4);
+        false ->
+            ok
     end.
 
 compile(State, AppInfo) ->
@@ -132,7 +161,9 @@ compile(State, Providers, AppInfo) ->
     AppInfo1 = rebar_hooks:run_all_hooks(AppDir, pre, ?PROVIDER,  Providers, AppInfo, State),
 
     AppInfo2 = rebar_hooks:run_all_hooks(AppDir, pre, ?ERLC_HOOK, Providers, AppInfo1, State),
-    rebar_erlc_compiler:compile(AppInfo2),
+
+    build_app(AppInfo2, State),
+
     AppInfo3 = rebar_hooks:run_all_hooks(AppDir, post, ?ERLC_HOOK, Providers, AppInfo2, State),
 
     AppInfo4 = rebar_hooks:run_all_hooks(AppDir, pre, ?APP_HOOK, Providers, AppInfo3, State),
@@ -141,11 +172,10 @@ compile(State, Providers, AppInfo) ->
     %% The rebar_otp_app compilation step is safe regarding the
     %% overall path management, so we can just load all plugins back
     %% in memory.
-    PluginDepsPaths = rebar_state:code_paths(State, all_plugin_deps),
-    code:add_pathsa(PluginDepsPaths),
+    rebar_paths:set_paths([plugins], State),
     AppFileCompileResult = rebar_otp_app:compile(State, AppInfo4),
-    %% Clean up after ourselves, leave things as they were.
-    rebar_utils:remove_from_code_path(PluginDepsPaths),
+    %% Clean up after ourselves, leave things as they were with deps first
+    rebar_paths:set_paths([deps], State),
 
     case AppFileCompileResult of
         {ok, AppInfo5} ->
@@ -161,9 +191,32 @@ compile(State, Providers, AppInfo) ->
 %% Internal functions
 %% ===================================================================
 
-update_code_paths(State, ProjectApps, DepsPaths) ->
+build_app(AppInfo, State) ->
+    case rebar_app_info:project_type(AppInfo) of
+        Type when Type =:= rebar3 ; Type =:= undefined ->
+            Compilers = rebar_state:compilers(State),
+            rebar_compiler:compile_all(Compilers, AppInfo);
+        Type ->
+            ProjectBuilders = rebar_state:project_builders(State),
+            case lists:keyfind(Type, 1, ProjectBuilders) of
+                {_, Module} ->
+                    %% load plugins since thats where project builders would be
+                    rebar_paths:set_paths([plugins, deps], State),
+                    Res = Module:build(AppInfo),
+                    rebar_paths:set_paths([deps], State),
+                    case Res of
+                        ok -> ok;
+                        {error, Reason} -> throw({error, {Module, Reason}})
+                    end;
+                _ ->
+                    throw(?PRV_ERROR({unknown_project_type, rebar_app_info:name(AppInfo), Type}))
+            end
+    end.
+
+update_code_paths(State, ProjectApps) ->
     ProjAppsPaths = paths_for_apps(ProjectApps),
     ExtrasPaths = paths_for_extras(State, ProjectApps),
+    DepsPaths = rebar_state:code_paths(State, all_deps),
     rebar_state:code_paths(State, all_deps, DepsPaths ++ ProjAppsPaths ++ ExtrasPaths).
 
 paths_for_apps(Apps) -> paths_for_apps(Apps, []).

src/rebar_prv_deps.erl

@@ -97,10 +97,11 @@ display_dep(_State, {Name, _Vsn, Source}) when is_tuple(Source) ->
 display_dep(_State, {Name, _Vsn, Source, _Opts}) when is_tuple(Source) ->
     ?CONSOLE("~ts* (~ts source)", [rebar_utils:to_binary(Name), type(Source)]);
 %% Locked
-display_dep(State, {Name, Source={pkg, _, Vsn}, Level}) when is_integer(Level) ->
+display_dep(State, {Name, _Source={pkg, _, Vsn}, Level}) when is_integer(Level) ->
     DepsDir = rebar_dir:deps_dir(State),
     AppDir = filename:join([DepsDir, rebar_utils:to_binary(Name)]),
-    NeedsUpdate = case rebar_fetch:needs_update(AppDir, Source, State) of
+    {ok, AppInfo} = rebar_app_info:discover(AppDir),
+    NeedsUpdate = case rebar_fetch:needs_update(AppInfo, State) of
         true -> "*";
         false -> ""
     end,
@@ -108,7 +109,8 @@ display_dep(State, {Name, Source={pkg, _, Vsn}, Level}) when is_integer(Level) -
 display_dep(State, {Name, Source, Level}) when is_tuple(Source), is_integer(Level) ->
     DepsDir = rebar_dir:deps_dir(State),
     AppDir = filename:join([DepsDir, rebar_utils:to_binary(Name)]),
-    NeedsUpdate = case rebar_fetch:needs_update(AppDir, Source, State) of
+    {ok, AppInfo} = rebar_app_info:discover(AppDir),
+    NeedsUpdate = case rebar_fetch:needs_update(AppInfo, State) of
         true -> "*";
         false -> ""
     end,

src/rebar_prv_deps_tree.erl

@@ -39,18 +39,16 @@ format_error(Reason) ->
 %% Internal functions
 
 print_deps_tree(SrcDeps, Verbose, State) ->
-    Resources = rebar_state:resources(State),
     D = lists:foldl(fun(App, Dict) ->
                             Name = rebar_app_info:name(App),
                             Vsn = rebar_app_info:original_vsn(App),
-                            AppDir = rebar_app_info:dir(App),
-                            Vsn1 = rebar_utils:vcs_vsn(Vsn, AppDir, Resources),
+                            Vsn1 = rebar_utils:vcs_vsn(App, Vsn, State),
                             Source  = rebar_app_info:source(App),
                             Parent = rebar_app_info:parent(App),
                             dict:append_list(Parent, [{Name, Vsn1, Source}], Dict)
                     end, dict:new(), SrcDeps),
     ProjectAppNames = [{rebar_app_info:name(App)
-                       ,rebar_utils:vcs_vsn(rebar_app_info:original_vsn(App), rebar_app_info:dir(App), Resources)
+                       ,rebar_utils:vcs_vsn(App, rebar_app_info:original_vsn(App), State)
                        ,project} || App <- rebar_state:project_apps(State)],
     case dict:find(root, D) of
         {ok, Children} ->

src/rebar_prv_dialyzer.erl

@@ -85,7 +85,8 @@ short_desc() ->
 do(State) ->
     maybe_fix_env(),
     ?INFO("Dialyzer starting, this may take a while...", []),
-    code:add_pathsa(rebar_state:code_paths(State, all_deps)),
+    rebar_paths:unset_paths([plugins], State), % no plugins in analysis
+    rebar_paths:set_paths([deps], State),
     Plt = get_plt(State),
 
     try
@@ -104,7 +105,7 @@ do(State) ->
         throw:{output_file_error, _, _} = Error ->
             ?PRV_ERROR(Error)
     after
-        rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default))
+        rebar_paths:set_paths([plugins,deps], State)
     end.
 
 %% This is used to workaround dialyzer quirk discussed here

src/rebar_prv_edoc.erl

@@ -32,7 +32,7 @@ init(State) ->
 -spec do(rebar_state:t()) ->
     {ok, rebar_state:t()} | {error, string()} | {error, {module(), any()}}.
 do(State) ->
-    code:add_pathsa(rebar_state:code_paths(State, all_deps)),
+    rebar_paths:set_paths([deps, plugins], State),
     ProjectApps = rebar_state:project_apps(State),
     Providers = rebar_state:providers(State),
     EdocOpts = rebar_state:get(State, edoc_opts, []),
@@ -64,7 +64,7 @@ do(State) ->
             {app_failed, AppName}
     end,
     rebar_hooks:run_all_hooks(Cwd, post, ?PROVIDER, Providers, State),
-    rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)),
+    rebar_paths:set_paths([plugins, deps], State),
     case Res of
         {app_failed, App} ->
             ?PRV_ERROR({app_failed, App});

src/rebar_prv_eunit.erl

@@ -54,7 +54,7 @@ do(State, Tests) ->
     ?INFO("Performing EUnit tests...", []),
 
     setup_name(State),
-    rebar_utils:update_code(rebar_state:code_paths(State, all_deps), [soft_purge]),
+    rebar_paths:set_paths([deps, plugins], State),
 
     %% Run eunit provider prehooks
     Providers = rebar_state:providers(State),
@@ -67,14 +67,14 @@ do(State, Tests) ->
                 {ok, State1} ->
                     %% Run eunit provider posthooks
                     rebar_hooks:run_project_and_app_hooks(Cwd, post, ?PROVIDER, Providers, State1),
-                    rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)),
+                    rebar_paths:set_paths([plugins, deps], State),
                     {ok, State1};
                 Error ->
-                    rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)),
+                    rebar_paths:set_paths([plugins, deps], State),
                     Error
             end;
         Error ->
-            rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)),
+            rebar_paths:set_paths([plugins, deps], State),
             Error
     end.
 

src/rebar_prv_install_deps.erl

@@ -259,9 +259,21 @@ update_seen_dep(AppInfo, _Profile, _Level, Deps, Apps, State, Upgrade, Seen, Loc
     %% If seen from lock file or user requested an upgrade
     %% don't print warning about skipping
     case lists:keymember(Name, 1, Locks) of
-        false when Upgrade -> ok;
-        false when not Upgrade -> warn_skip_deps(AppInfo, State);
-        true -> ok
+        false when Upgrade ->
+            ok;
+        false when not Upgrade ->
+            {ok, SeenApp} = rebar_app_utils:find(Name, Apps),
+            Source = rebar_app_info:source(AppInfo),
+            case rebar_app_info:source(SeenApp) of
+                Source ->
+                    %% dep is the same version and checksum as the one we already saw.
+                    %% meaning there is no conflict, so don't warn about it.
+                    skip;
+                _ ->
+                    warn_skip_deps(Name, Source, State)
+            end;
+        true ->
+            ok
     end,
     {Deps, Apps, State, Seen}.
 
@@ -277,10 +289,8 @@ update_unseen_dep(AppInfo, Profile, Level, Deps, Apps, State, Upgrade, Seen, Loc
 -spec handle_dep(rebar_state:t(), atom(), file:filename_all(), rebar_app_info:t(), list(), integer()) -> {rebar_app_info:t(), [rebar_app_info:t()], rebar_state:t()}.
 handle_dep(State, Profile, DepsDir, AppInfo, Locks, Level) ->
     Name = rebar_app_info:name(AppInfo),
-    C = rebar_config:consult(rebar_app_info:dir(AppInfo)),
 
-    AppInfo0 = rebar_app_info:update_opts(AppInfo, rebar_app_info:opts(AppInfo), C),
-    AppInfo1 = rebar_app_info:apply_overrides(rebar_app_info:get(AppInfo, overrides, []), AppInfo0),
+    AppInfo1 = rebar_app_info:apply_overrides(rebar_app_info:get(AppInfo, overrides, []), AppInfo),
     AppInfo2 = rebar_app_info:apply_profiles(AppInfo1, [default, prod]),
 
     Plugins = rebar_app_info:get(AppInfo2, plugins, []),
@@ -297,34 +307,33 @@ handle_dep(State, Profile, DepsDir, AppInfo, Locks, Level) ->
     AppInfo4 = rebar_app_info:deps(AppInfo3, rebar_state:deps_names(Deps)),
 
     %% Keep all overrides from the global config and this dep when parsing its deps
-    Overrides = rebar_app_info:get(AppInfo0, overrides, []),
+    Overrides = rebar_app_info:get(AppInfo, overrides, []),
     Deps1 = rebar_app_utils:parse_deps(Name, DepsDir, Deps, rebar_state:set(State, overrides, Overrides)
                                       ,Locks, Level+1),
     {AppInfo4, Deps1, State1}.
 
 -spec maybe_fetch(rebar_app_info:t(), atom(), boolean(),
-                  sets:set(binary()), rebar_state:t()) -> {boolean(), rebar_app_info:t()}.
+                  sets:set(binary()), rebar_state:t()) -> {ok, rebar_app_info:t()}.
 maybe_fetch(AppInfo, Profile, Upgrade, Seen, State) ->
     AppDir = rebar_utils:to_list(rebar_app_info:dir(AppInfo)),
     %% Don't fetch dep if it exists in the _checkouts dir
     case rebar_app_info:is_checkout(AppInfo) of
         true ->
-            {false, AppInfo};
+            {ok, AppInfo};
         false ->
-            case rebar_app_discover:find_app(AppInfo, AppDir, all) of
+            case rebar_app_info:is_available(AppInfo) of
                 false ->
-                    true = fetch_app(AppInfo, AppDir, State),
-                    maybe_symlink_default(State, Profile, AppDir, AppInfo),
-                    {true, rebar_app_info:valid(update_app_info(AppDir, AppInfo), false)};
-                {true, AppInfo1} ->
-                    case sets:is_element(rebar_app_info:name(AppInfo1), Seen) of
+                    AppInfo1 = fetch_app(AppInfo, State),
+                    maybe_symlink_default(State, Profile, AppDir, AppInfo1),
+                    {ok, rebar_app_info:is_available(rebar_app_info:valid(AppInfo1, false), true)};
+                true ->
+                    case sets:is_element(rebar_app_info:name(AppInfo), Seen) of
                         true ->
-                            {false, AppInfo1};
+                            {ok, AppInfo};
                         false ->
-                            maybe_symlink_default(State, Profile, AppDir, AppInfo1),
-                            MaybeUpgrade = maybe_upgrade(AppInfo, AppDir, Upgrade, State),
-                            AppInfo2 = update_app_info(AppDir, AppInfo1),
-                            {MaybeUpgrade, AppInfo2}
+                            maybe_symlink_default(State, Profile, AppDir, AppInfo),
+                            AppInfo1 = maybe_upgrade(AppInfo, AppDir, Upgrade, State),
+                            {ok, AppInfo1}
                     end
             end
     end.
@@ -372,52 +381,37 @@ make_relative_to_root(State, Path) when is_list(Path) ->
     Root = rebar_dir:root_dir(State),
     rebar_dir:make_relative_path(Path, Root).
 
-fetch_app(AppInfo, AppDir, State) ->
+fetch_app(AppInfo, State) ->
     ?INFO("Fetching ~ts (~p)", [rebar_app_info:name(AppInfo),
-                               format_source(rebar_app_info:source(AppInfo))]),
-    Source = rebar_app_info:source(AppInfo),
-    true = rebar_fetch:download_source(AppDir, Source, State).
-
-format_source({pkg, Name, Vsn, _Hash}) -> {pkg, Name, Vsn};
-format_source(Source) -> Source.
-
-%% This is called after the dep has been downloaded and unpacked, if it hadn't been already.
-%% So this is the first time for newly downloaded apps that its .app/.app.src data can
-%% be read in an parsed.
-update_app_info(AppDir, AppInfo) ->
-    case rebar_app_discover:find_app(AppInfo, AppDir, all) of
-        {true, AppInfo1} ->
-            AppInfo1;
-        false ->
-            throw(?PRV_ERROR({dep_app_not_found, AppDir, rebar_app_info:name(AppInfo)}))
-    end.
+                                rebar_resource_v2:format_source(rebar_app_info:source(AppInfo))]),
+    rebar_fetch:download_source(AppInfo, State).
 
-maybe_upgrade(AppInfo, AppDir, Upgrade, State) ->
-    Source = rebar_app_info:source(AppInfo),
+maybe_upgrade(AppInfo, _AppDir, Upgrade, State) ->
     case Upgrade orelse rebar_app_info:is_lock(AppInfo) of
         true ->
-            case rebar_fetch:needs_update(AppDir, Source, State) of
+            case rebar_fetch:needs_update(AppInfo, State) of
                 true ->
-                    ?INFO("Upgrading ~ts (~p)", [rebar_app_info:name(AppInfo), rebar_app_info:source(AppInfo)]),
-                    true = rebar_fetch:download_source(AppDir, Source, State);
+                    ?INFO("Upgrading ~ts (~p)", [rebar_app_info:name(AppInfo),
+                                                 rebar_resource_v2:format_source(rebar_app_info:source(AppInfo))]),
+                    rebar_fetch:download_source(AppInfo, State);
                 false ->
                     case Upgrade of
                         true ->
                             ?INFO("No upgrade needed for ~ts", [rebar_app_info:name(AppInfo)]),
-                            false;
+                            AppInfo;
                         false ->
-                            false
+                            AppInfo
                     end
             end;
         false ->
-            false
+            AppInfo
     end.
 
-warn_skip_deps(AppInfo, State) ->
+warn_skip_deps(Name, Source, State) ->
     Msg = "Skipping ~ts (from ~p) as an app of the same name "
           "has already been fetched",
-    Args = [rebar_app_info:name(AppInfo),
-            rebar_app_info:source(AppInfo)],
+    Args = [Name,
+            rebar_resource_v2:format_source(Source)],
     case rebar_state:get(State, deps_error_on_conflict, false) of
         false ->
             case rebar_state:get(State, deps_warning_on_conflict, true) of

src/rebar_prv_local_upgrade.erl

@@ -77,7 +77,7 @@ get_md5(Rebar3Path) ->
 maybe_fetch_rebar3(Rebar3Md5) ->
     TmpDir = ec_file:insecure_mkdtemp(),
     TmpFile = filename:join(TmpDir, "rebar3"),
-    case rebar_pkg_resource:request("https://s3.amazonaws.com/rebar3/rebar3", Rebar3Md5) of
+    case request("https://s3.amazonaws.com/rebar3/rebar3", Rebar3Md5) of
         {ok, Binary, ETag} ->
             file:write_file(TmpFile, Binary),
             case etag(TmpFile) of
@@ -101,3 +101,29 @@ etag(Path) ->
          {error, _} ->
              false
      end.
+
+-spec request(Url, ETag) -> Res when
+      Url :: string(),
+      ETag :: false | string(),
+      Res :: 'error' | {ok, cached} | {ok, any(), string()}.
+request(Url, ETag) ->
+    HttpOptions = [{ssl, rebar_utils:ssl_opts(Url)},
+                   {relaxed, true} | rebar_utils:get_proxy_auth()],
+    case httpc:request(get, {Url, [{"if-none-match", "\"" ++ ETag ++ "\""}
+                                   || ETag =/= false] ++
+                                 [{"User-Agent", rebar_utils:user_agent()}]},
+                       HttpOptions, [{body_format, binary}], rebar) of
+        {ok, {{_Version, 200, _Reason}, Headers, Body}} ->
+            ?DEBUG("Successfully downloaded ~ts", [Url]),
+            {"etag", ETag1} = lists:keyfind("etag", 1, Headers),
+            {ok, Body, rebar_string:trim(ETag1, both, [$"])};
+        {ok, {{_Version, 304, _Reason}, _Headers, _Body}} ->
+            ?DEBUG("Cached copy of ~ts still valid", [Url]),
+            {ok, cached};
+        {ok, {{_Version, Code, _Reason}, _Headers, _Body}} ->
+            ?DEBUG("Request to ~p failed: status code ~p", [Url, Code]),
+            error;
+        {error, Reason} ->
+            ?DEBUG("Request to ~p failed: ~p", [Url, Reason]),
+            error
+    end.

src/rebar_prv_lock.erl

@@ -54,12 +54,9 @@ format_error(Reason) ->
 build_locks(State) ->
     AllDeps = rebar_state:lock(State),
     [begin
-        Dir = rebar_app_info:dir(Dep),
-        Source = rebar_app_info:source(Dep),
-
         %% If source is tuple it is a source dep
         %% e.g. {git, "git://github.com/ninenines/cowboy.git", "master"}
-        {rebar_app_info:name(Dep)
-         class="p">,rebar_fetch:lock_source(Dir, Source, State)
-         class="p">,rebar_app_info:dep_level(Dep)}
+        {rebar_app_info:name(Dep),
+         rebar_fetch:lock_source(Dep, State),
+         rebar_app_info:dep_level(Dep)}
      end || Dep <- AllDeps, not(rebar_app_info:is_checkout(Dep))].

src/rebar_prv_packages.erl

@@ -15,53 +15,75 @@
 
 -spec init(rebar_state:t()) -> {ok, rebar_state:t()}.
 init(State) ->
-    State1 = rebar_state:add_provider(State, providers:create([{name, ?PROVIDER},
-                                                               {module, ?MODULE},
-                                                               {bare, true},
-                                                               {deps, ?DEPS},
-                                                               {example, "rebar3 pkgs"},
-                                                               {short_desc, "List available packages."},
-                                                               {desc, info("List available packages")},
-                                                               {opts, []}])),
+    State1 = rebar_state:add_provider(State,
+                                      providers:create([{name, ?PROVIDER},
+                                                        {module, ?MODULE},
+                                                        {bare, true},
+                                                        {deps, ?DEPS},
+                                                        {example, "rebar3 pkgs elli"},
+                                                        {short_desc, "List information for a package."},
+                                                        {desc, info("List information for a package")},
+                                                        {opts, [{package, undefined, undefined, string,
+                                                                 "Package to fetch information for."}]}])),
     {ok, State1}.
 
 -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}.
 do(State) ->
-    rebar_packages:packages(State),
-    case rebar_state:command_args(State) of
-        [Name] ->
-            print_packages(get_packages(rebar_utils:to_binary(Name)));
-        _ ->
-            print_packages(sort_packages())
-    end,
-    {ok, State}.
+    {Args, _} = rebar_state:command_parsed_args(State),
+    case proplists:get_value(package, Args, undefined) of
+        undefined ->
+            ?PRV_ERROR(no_package_arg);
+        Name ->
+            Resources = rebar_state:resources(State),
+            #{repos := Repos} = rebar_resource_v2:find_resource_state(pkg, Resources),
+            Results = get_package(rebar_utils:to_binary(Name), Repos),
+            case lists:all(fun({_, {error, not_found}}) -> true; (_) -> false end, Results) of
+                true ->
+                    ?PRV_ERROR({not_found, Name});
+                false ->
+                    [print_packages(Result) || Result <- Results],
+                    {ok, State}
+            end
+    end.
 
--spec format_error(any()) -> iolist().
-format_error(load_registry_fail) ->
-    "Failed to load package regsitry. Try running 'rebar3 update' to fix".
+-spec get_package(binary(), [map()]) -> [{binary(), {ok, map()} | {error, term()}}].
+get_package(Name, Repos) ->
+    lists:foldl(fun(RepoConfig, Acc) ->
+                        [{maps:get(name, RepoConfig), rebar_packages:get(RepoConfig, Name)} | Acc]
+                end, [], Repos).
 
-print_packages(Pkgs) ->
-    orddict:map(fun(Name, Vsns) ->
-                        SortedVsns = lists:sort(fun(A, B) ->
-                                                        ec_semver:lte(ec_semver:parse(A)
-                                                                     ,ec_semver:parse(B))
-                                                end, Vsns),
-                        VsnStr = join(SortedVsns, <<", ">>),
-                        ?CONSOLE("~ts:~n    Versions: ~ts~n", [Name, VsnStr])
-                end, Pkgs).
 
-sort_packages() ->
-    ets:foldl(fun({package_index_version, _}, Acc) ->
-                      Acc;
-                 ({Pkg, Vsns}, Acc) ->
-                      orddict:store(Pkg, Vsns, Acc);
-                 (_, Acc) ->
-                      Acc
-              end, orddict:new(), ?PACKAGE_TABLE).
+-spec format_error(any()) -> iolist().
+format_error(no_package_arg) ->
+    "Missing package argument to `rebar3 pkgs` command.";
+format_error({not_found, Name}) ->
+    io_lib:format("Package ~ts not found in any repo.", [Name]);
+format_error(unknown) ->
+    "Something went wrong with fetching package metadata.".
 
-get_packages(Name) ->
-    ets:lookup(?PACKAGE_TABLE, Name).
 
+print_packages({RepoName, {error, not_found}}) ->
+    ?CONSOLE("~ts: Package not found in this repo.~n", [RepoName]);
+print_packages({RepoName, {error, _}}) ->
+    ?CONSOLE("~ts: Error fetching from this repo.~n", [RepoName]);
+print_packages({RepoName, {ok, #{<<"name">> := Name,
+                                 <<"meta">> := Meta,
+                                 <<"releases">> := Releases}}}) ->
+    Description = maps:get(<<"description">>, Meta, ""),
+    Licenses = join(maps:get(<<"licenses">>, Meta, []), <<", ">>),
+    Links = join_map(maps:get(<<"links">>, Meta, []), <<"\n        ">>),
+    Maintainers = join(maps:get(<<"maintainers">>, Meta, []), <<", ">>),
+    Versions = [V || #{<<"version">> := V} <- Releases],
+    VsnStr = join(Versions, <<", ">>),
+    ?CONSOLE("~ts:~n"
+             "    Name: ~ts~n"
+             "    Description: ~ts~n"
+             "    Licenses: ~ts~n"
+             "    Maintainers: ~ts~n"
+             "    Links:~n        ~ts~n"
+             "    Versions: ~ts~n", [RepoName, Name, Description, Licenses, Maintainers, Links, VsnStr]);
+print_packages(_) ->
+    ok.
 
 -spec join([binary()], binary()) -> binary().
 join([Bin], _Sep) ->
@@ -69,6 +91,14 @@ join([Bin], _Sep) ->
 join([Bin | T], Sep) ->
     <<Bin/binary, Sep/binary, (join(T, Sep))/binary>>.
 
+-spec join_map(map(), binary()) -> binary().
+join_map(Map, Sep) ->
+    join_tuple_list(maps:to_list(Map), Sep).
+
+join_tuple_list([{K, V}], _Sep) ->
+    <<K/binary, ": ", V/binary>>;
+join_tuple_list([{K, V} | T], Sep) ->
+    <<K/binary, ": ", V/binary, Sep/binary, (join_tuple_list(T, Sep))/binary>>.
 
 info(Description) ->
     io_lib:format("~ts.~n", [Description]).

src/rebar_prv_plugins.erl

@@ -36,7 +36,7 @@ do(State) ->
     GlobalPlugins = rebar_state:get(GlobalConfig, plugins, []),
     GlobalSrcDirs = rebar_state:get(GlobalConfig, src_dirs, ["src"]),
     GlobalPluginsDir = filename:join([rebar_dir:global_cache_dir(rebar_state:opts(State)), "plugins", "*"]),
-    GlobalApps = rebar_app_discover:find_apps([GlobalPluginsDir], GlobalSrcDirs, all),
+    GlobalApps = rebar_app_discover:find_apps([GlobalPluginsDir], GlobalSrcDirs, all, State),
     display_plugins("Global plugins", GlobalApps, GlobalPlugins),
 
     RebarOpts = rebar_state:opts(State),
@@ -44,7 +44,7 @@ do(State) ->
     Plugins = rebar_state:get(State, plugins, []),
     PluginsDirs = filelib:wildcard(filename:join(rebar_dir:plugins_dir(State), "*")),
     CheckoutsDirs = filelib:wildcard(filename:join(rebar_dir:checkouts_dir(State), "*")),
-    Apps = rebar_app_discover:find_apps(CheckoutsDirs++PluginsDirs, SrcDirs, all),
+    Apps = rebar_app_discover:find_apps(CheckoutsDirs++PluginsDirs, SrcDirs, all, State),
     display_plugins("Local plugins", Apps, Plugins),
     {ok, State}.
 

src/rebar_prv_repos.erl

@@ -0,0 +1,47 @@
+%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*-
+%% ex: ts=4 sw=4 et
+
+-module(rebar_prv_repos).
+
+-behaviour(provider).
+
+-export([init/1,
+         do/1,
+         format_error/1]).
+
+-include("rebar.hrl").
+
+-define(PROVIDER, repos).
+-define(DEPS, []).
+
+%% ===================================================================
+%% Public API
+%% ===================================================================
+
+-spec init(rebar_state:t()) -> {ok, rebar_state:t()}.
+init(State) ->
+    Provider = providers:create(
+        [{name, ?PROVIDER},
+         {module, ?MODULE},
+         {bare, false},
+         {deps, ?DEPS},
+         {example, "rebar3 repos"},
+         {short_desc, "Print current package repository configuration"},
+         {desc, "Display repository configuration for debugging purpose"},
+         {opts, []}]),
+    State1 = rebar_state:add_provider(State, Provider),
+    {ok, State1}.
+
+-spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}.
+do(State) ->
+    Resources = rebar_state:resources(State),
+    #{repos := Repos} = rebar_resource_v2:find_resource_state(pkg, Resources),
+
+    ?CONSOLE("Repos:", []),
+    %%TODO: do some formatting
+    ?CONSOLE("~p", [Repos]),
+    {ok, State}.
+
+-spec format_error(any()) -> iolist().
+format_error(Reason) ->
+    io_lib:format("~p", [Reason]).

src/rebar_prv_shell.erl

@@ -40,6 +40,8 @@
 -define(PROVIDER, shell).
 -define(DEPS, [compile]).
 
+-dialyzer({nowarn_function, rewrite_leaders/2}).
+
 %% ===================================================================
 %% Public API
 %% ===================================================================
@@ -385,7 +387,7 @@ reread_config(AppsToStart, State) ->
                     lists:member(App, Running),
                     lists:member(App, AppsToStart),
                     not lists:member(App, BlackList)],
-            _ = rebar_utils:reread_config(ConfigList),
+            _ = rebar_utils:reread_config(ConfigList, [update_logger]),
             ok
     end.
 

src/rebar_prv_update.erl

@@ -9,12 +9,6 @@
          do/1,
          format_error/1]).
 
--export([hex_to_index/1]).
-
--ifdef(TEST).
--export([cmp_/6, cmpl_/6, valid_vsn/1]).
--endif.
-
 -include("rebar.hrl").
 -include_lib("providers/include/providers.hrl").
 
@@ -39,44 +33,13 @@ init(State) ->
 
 -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}.
 do(State) ->
-    try
-        case rebar_packages:registry_dir(State) of
-            {ok, RegistryDir} ->
-                filelib:ensure_dir(filename:join(RegistryDir, "dummy")),
-                HexFile = filename:join(RegistryDir, "registry"),
-                ?INFO("Updating package registry...", []),
-                TmpDir = ec_file:insecure_mkdtemp(),
-                TmpFile = filename:join(TmpDir, "packages.gz"),
-
-                CDN = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN),
-                case rebar_utils:url_append_path(CDN, ?REMOTE_REGISTRY_FILE) of
-                    {ok, Url} ->
-                        HttpOptions = [{relaxed, true} | rebar_utils:get_proxy_auth()],
-                        ?DEBUG("Fetching registry from ~p", [Url]),
-                        case httpc:request(get, {Url, [{"User-Agent", rebar_utils:user_agent()}]},
-                                           HttpOptions, [{stream, TmpFile}, {sync, true}],
-                                           rebar) of
-                            {ok, saved_to_file} ->
-                                {ok, Data} = file:read_file(TmpFile),
-                                Unzipped = zlib:gunzip(Data),
-                                ok = file:write_file(HexFile, Unzipped),
-                                ?INFO("Writing registry to ~ts", [HexFile]),
-                                hex_to_index(State),
-                                {ok, State};
-                            _ ->
-                                ?PRV_ERROR(package_index_download)
-                        end;
-                    _ ->
-                        ?PRV_ERROR({package_parse_cdn, CDN})
-                end;
-            {uri_parse_error, CDN} ->
-                ?PRV_ERROR({package_parse_cdn, CDN})
-        end
-    catch
-        ?WITH_STACKTRACE(_E, C, S)
-            ?DEBUG("Error creating package index: ~p ~p", [C, S]),
-            throw(?PRV_ERROR(package_index_write))
-    end.
+    Names = rebar_packages:get_all_names(State),
+    Resources = rebar_state:resources(State),
+    #{repos := RepoConfigs} = rebar_resource_v2:find_resource_state(pkg, Resources),
+    [[update_package(Name, RepoConfig, State)
+      || Name <- Names]
+     || RepoConfig <- RepoConfigs],
+    {ok, State}.
 
 -spec format_error(any()) -> iolist().
 format_error({package_parse_cdn, Uri}) ->
@@ -86,186 +49,11 @@ format_error(package_index_download) ->
 format_error(package_index_write) ->
     "Failed to write package index.".
 
-is_supported(<<"make">>) -> true;
-is_supported(<<"rebar">>) -> true;
-is_supported(<<"rebar3">>) -> true;
-is_supported(_) -> false.
-
-hex_to_index(State) ->
-    {ok, RegistryDir} = rebar_packages:registry_dir(State),
-    HexFile = filename:join(RegistryDir, "registry"),
-    try ets:file2tab(HexFile) of
-        {ok, Registry} ->
-            try
-                PackageIndex = filename:join(RegistryDir, "packages.idx"),
-                ?INFO("Generating package index...", []),
-                (catch ets:delete(?PACKAGE_TABLE)),
-                ets:new(?PACKAGE_TABLE, [named_table, public]),
-                ets:foldl(fun({{Pkg, PkgVsn}, [Deps, Checksum, BuildTools | _]}, _) when is_list(BuildTools) ->
-                                  case lists:any(fun is_supported/1, BuildTools) of
-                                      true ->
-                                          DepsList = update_deps_list(Pkg, PkgVsn, Deps, Registry, State),
-                                          HashedDeps = update_deps_hashes(DepsList),
-                                          ets:insert(?PACKAGE_TABLE, {{Pkg, PkgVsn}, HashedDeps, Checksum});
-                                      false ->
-                                          true
-                                  end;
-                             (_, _) ->
-                                  true
-                          end, true, Registry),
-
-                ets:foldl(fun({Pkg, [[]]}, _) when is_binary(Pkg) ->
-                                  true;
-                             ({Pkg, [Vsns=[_Vsn | _Rest]]}, _) when is_binary(Pkg) ->
-                                  %% Verify the package is of the right build tool by checking if the first
-                                  %% version exists in the table from the foldl above
-                                  case [V || V <- Vsns, ets:member(?PACKAGE_TABLE, {Pkg, V})] of
-                                      [] ->
-                                          true;
-                                      Vsns1 ->
-                                          ets:insert(?PACKAGE_TABLE, {Pkg, Vsns1})
-                                  end;
-                             (_, _) ->
-                                  true
-                          end, true, Registry),
-                ets:insert(?PACKAGE_TABLE, {package_index_version, ?PACKAGE_INDEX_VERSION}),
-                ?INFO("Writing index to ~ts", [PackageIndex]),
-                ets:tab2file(?PACKAGE_TABLE, PackageIndex),
-                true
-            after
-                catch ets:delete(Registry)
-            end;
-        {error, Reason} ->
-            ?DEBUG("Error loading package registry: ~p", [Reason]),
-            false
-    catch
-        _:_ ->
-            fail
-    end.
-
-update_deps_list(Pkg, PkgVsn, Deps, HexRegistry, State) ->
-    lists:foldl(fun([Dep, DepVsn, false, AppName | _], DepsListAcc) ->
-                        Dep1 = {Pkg, PkgVsn, Dep, AppName},
-                        case {valid_vsn(DepVsn), DepVsn} of
-                            %% Those are all not perfectly implemented!
-                            %% and doubled since spaces seem not to be
-                            %% enforced
-                            {false, Vsn} ->
-                                ?DEBUG("[~ts:~ts], Bad dependency version for ~ts: ~ts.",
-                                       [Pkg, PkgVsn, Dep, Vsn]),
-                                DepsListAcc;
-                            {_, <<"~>", Vsn/binary>>} ->
-                                highest_matching(Dep1, rm_ws(Vsn), HexRegistry,
-                                                 State, DepsListAcc);
-                            {_, <<">=", Vsn/binary>>} ->
-                                cmp(Dep1, rm_ws(Vsn), HexRegistry, State,
-                                    DepsListAcc, fun ec_semver:gte/2);
-                            {_, <<">", Vsn/binary>>} ->
-                                cmp(Dep1, rm_ws(Vsn), HexRegistry, State,
-                                    DepsListAcc, fun ec_semver:gt/2);
-                            {_, <<"<=", Vsn/binary>>} ->
-                                cmpl(Dep1, rm_ws(Vsn), HexRegistry, State,
-                                     DepsListAcc, fun ec_semver:lte/2);
-                            {_, <<"<", Vsn/binary>>} ->
-                                cmpl(Dep1, rm_ws(Vsn), HexRegistry, State,
-                                     DepsListAcc, fun ec_semver:lt/2);
-                            {_, <<"==", Vsn/binary>>} ->
-                                [{AppName, {pkg, Dep, Vsn, undefined}} | DepsListAcc];
-                            {_, Vsn} ->
-                                [{AppName, {pkg, Dep, Vsn, undefined}} | DepsListAcc]
-                        end;
-                   ([_Dep, _DepVsn, true, _AppName | _], DepsListAcc) ->
-                        DepsListAcc
-                end, [], Deps).
-
-update_deps_hashes(List) ->
-    [{Name, {pkg, PkgName, Vsn, lookup_hash(PkgName, Vsn, Hash)}}
-     || {Name, {pkg, PkgName, Vsn, Hash}} <- List].
-
-lookup_hash(Name, Vsn, undefined) ->
-    try
-        ets:lookup_element(?PACKAGE_TABLE, {Name, Vsn}, 3)
-    catch
-        _:_ ->
-            undefined
-    end;
-lookup_hash(_, _, Hash) ->
-    Hash.
-
-
-rm_ws(<<" ", R/binary>>) ->
-    rm_ws(R);
-rm_ws(R) ->
-    R.
-
-valid_vsn(Vsn) ->
-    %% Regepx from https://github.com/sindresorhus/semver-regex/blob/master/index.js
-    SemVerRegExp = "v?(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)(\\.(0|[1-9][0-9]*))?"
-        "(-[0-9a-z-]+(\\.[0-9a-z-]+)*)?(\\+[0-9a-z-]+(\\.[0-9a-z-]+)*)?",
-    SupportedVersions = "^(>=?|<=?|~>|==)?\\s*" ++ SemVerRegExp ++ "$",
-    re:run(Vsn, SupportedVersions, [unicode]) =/= nomatch.
-
-highest_matching({Pkg, PkgVsn, Dep, App}, Vsn, HexRegistry, State, DepsListAcc) ->
-    case rebar_packages:find_highest_matching_(Pkg, PkgVsn, Dep, Vsn, HexRegistry, State) of
-        {ok, HighestDepVsn} ->
-            [{App, {pkg, Dep, HighestDepVsn, undefined}} | DepsListAcc];
-        none ->
-            ?DEBUG("[~ts:~ts] Missing registry entry for package ~ts. Try to fix with `rebar3 update`",
-                  [Pkg, PkgVsn, Dep]),
-            DepsListAcc
-    end.
-
-cmp({_Pkg, _PkgVsn, Dep, _App} = Dep1, Vsn, HexRegistry, State, DepsListAcc, CmpFun) ->
-    {ok, Vsns}  = rebar_packages:find_all(Dep, HexRegistry, State),
-    cmp_(undefined, Vsn, Vsns, DepsListAcc, Dep1, CmpFun).
-
-
-cmp_(undefined, _MinVsn, [], DepsListAcc, {Pkg, PkgVsn, Dep, _App}, _CmpFun) ->
-    ?DEBUG("[~ts:~ts] Missing registry entry for package ~ts. Try to fix with `rebar3 update`",
-           [Pkg, PkgVsn, Dep]),
-    DepsListAcc;
-cmp_(HighestDepVsn, _MinVsn, [], DepsListAcc, {_Pkg, _PkgVsn, Dep, App}, _CmpFun) ->
-    [{App, {pkg, Dep, HighestDepVsn, undefined}} | DepsListAcc];
-
-cmp_(BestMatch, MinVsn, [Vsn | R], DepsListAcc, Dep, CmpFun) ->
-    case CmpFun(Vsn, MinVsn) of
-        true ->
-            cmp_(Vsn, Vsn, R, DepsListAcc, Dep, CmpFun);
-        false  ->
-            cmp_(BestMatch, MinVsn, R, DepsListAcc, Dep, CmpFun)
-    end.
-
-%% We need to treat this differently since we want a version that is LOWER but
-%% the higest possible one.
-cmpl({_Pkg, _PkgVsn, Dep, _App} = Dep1, Vsn, HexRegistry, State, DepsListAcc, CmpFun) ->
-    {ok, Vsns}  = rebar_packages:find_all(Dep, HexRegistry, State),
-    cmpl_(undefined, Vsn, Vsns, DepsListAcc, Dep1, CmpFun).
-
-cmpl_(undefined, _MaxVsn, [], DepsListAcc, {Pkg, PkgVsn, Dep, _App}, _CmpFun) ->
-    ?DEBUG("[~ts:~ts] Missing registry entry for package ~ts. Try to fix with `rebar3 update`",
-           [Pkg, PkgVsn, Dep]),
-    DepsListAcc;
-
-cmpl_(HighestDepVsn, _MaxVsn, [], DepsListAcc, {_Pkg, _PkgVsn, Dep, App}, _CmpFun) ->
-    [{App, {pkg, Dep, HighestDepVsn, undefined}} | DepsListAcc];
-
-cmpl_(undefined, MaxVsn, [Vsn | R], DepsListAcc, Dep, CmpFun) ->
-    case CmpFun(Vsn, MaxVsn) of
-        true ->
-            cmpl_(Vsn, MaxVsn, R, DepsListAcc, Dep, CmpFun);
-        false  ->
-            cmpl_(undefined, MaxVsn, R, DepsListAcc, Dep, CmpFun)
-    end;
 
-cmpl_(BestMatch, MaxVsn, [Vsn | R], DepsListAcc, Dep, CmpFun) ->
-    case CmpFun(Vsn, MaxVsn) of
-        true ->
-            case ec_semver:gte(Vsn, BestMatch) of
-                true ->
-                    cmpl_(Vsn, MaxVsn, R, DepsListAcc, Dep, CmpFun);
-                false ->
-                    cmpl_(BestMatch, MaxVsn, R, DepsListAcc, Dep, CmpFun)
-            end;
-        false  ->
-            cmpl_(BestMatch, MaxVsn, R, DepsListAcc, Dep, CmpFun)
+update_package(Name, RepoConfig, State) ->
+    case rebar_packages:update_package(Name, RepoConfig, State) of
+        fail ->
+            ?WARN("Failed to fetch updates for package ~ts from repo ~ts", [Name, maps:get(name, RepoConfig)]);
+        _ ->
+            ok
     end.

src/rebar_prv_upgrade.erl

@@ -82,17 +82,22 @@ do_(State) ->
     Deps = [Dep || Dep <- TopDeps ++ ProfileDeps, % TopDeps > ProfileDeps
                    is_atom(Dep) orelse is_atom(element(1, Dep))],
     Names = parse_names(rebar_utils:to_binary(proplists:get_value(package, Args, <<"">>)), Locks),
+
     DepsDict = deps_dict(rebar_state:all_deps(State)),
     AltDeps = find_non_default_deps(Deps, State),
     FilteredNames = cull_default_names_if_profiles(Names, Deps, State),
     case prepare_locks(FilteredNames, Deps, Locks, [], DepsDict, AltDeps) of
         {error, Reason} ->
             {error, Reason};
-        {Locks0, _Unlocks0} ->
+        {Locks0, Unlocks0} ->
             Deps0 = top_level_deps(Deps, Locks),
             State1 = rebar_state:set(State, {deps, default}, Deps0),
             DepsDir = rebar_prv_install_deps:profile_dep_dir(State, default),
             D = rebar_app_utils:parse_deps(root, DepsDir, Deps0, State1, Locks0, 0),
+
+            %% first update the package index for the packages to be upgraded
+            update_pkg_deps(Unlocks0, D, State1),
+
             State2 = rebar_state:set(State1, {parsed_deps, default}, D),
             State3 = rebar_state:set(State2, {locks, default}, Locks0),
             State4 = rebar_state:set(State3, upgrade, true),
@@ -121,6 +126,34 @@ format_error({transitive_dependency, Name}) ->
 format_error(Reason) ->
     io_lib:format("~p", [Reason]).
 
+%% fetch updates for package deps that have been unlocked for upgrade
+update_pkg_deps([], _, _) ->
+    ok;
+update_pkg_deps([{Name, _, _} | Rest], AppInfos, State) ->
+    case rebar_app_utils:find(Name, AppInfos) of
+        {ok, AppInfo} ->
+            case element(1, rebar_app_info:source(AppInfo)) of
+                pkg ->
+                    Resources = rebar_state:resources(State),
+                    #{repos := RepoConfigs} = rebar_resource_v2:find_resource_state(pkg, Resources),
+                    [update_package(Name, RepoConfig, State) || RepoConfig <- RepoConfigs];
+                _ ->
+                    skip
+            end;
+        _ ->
+            %% this should be impossible...
+            skip
+    end,
+    update_pkg_deps(Rest, AppInfos, State).
+
+update_package(Name, RepoConfig, State) ->
+    case rebar_packages:update_package(Name, RepoConfig, State) of
+        fail ->
+            ?WARN("Failed to fetch updates for package ~ts from repo ~ts", [Name, maps:get(name, RepoConfig)]);
+        _ ->
+            ok
+    end.
+
 parse_names(Bin, Locks) ->
     case lists:usort(re:split(Bin, <<" *, *">>, [trim, unicode])) of
         %% Nothing submitted, use *all* apps

src/rebar_prv_xref.erl

@@ -36,8 +36,7 @@ init(State) ->
 
 -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}.
 do(State) ->
-    OldPath = code:get_path(),
-    code:add_pathsa(rebar_state:code_paths(State, all_deps)),
+    rebar_paths:set_paths([deps], State),
     XrefChecks = prepare(State),
     XrefIgnores = rebar_state:get(State, xref_ignores, []),
     %% Run xref checks
@@ -48,7 +47,6 @@ do(State) ->
     QueryChecks = rebar_state:get(State, xref_queries, []),
     QueryResults = lists:foldl(fun check_query/2, [], QueryChecks),
     stopped = xref:stop(xref),
-    rebar_utils:cleanup_code_path(OldPath),
     case XrefResults =:= [] andalso QueryResults =:= [] of
         true ->
             {ok, State};

src/rebar_relx.erl

@@ -40,7 +40,8 @@ do(Module, Command, Provider, State) ->
                           ,{caller, api}
                           ,{log_level, LogLevel} | output_dir(OutputDir, Options)] ++ ErlOpts, AllOptions);
             Config ->
-                Config1 = merge_overlays(Config),
+                Config1 = [{overlay_vars, [{base_dir, rebar_dir:base_dir(State)}]}
+                           | merge_overlays(Config)],
                 relx:main([{lib_dirs, LibDirs}
                           ,{config, Config1}
                           ,{caller, api}

src/rebar_resource.erl

@@ -2,23 +2,53 @@
 %% ex: ts=4 sw=4 et
 -module(rebar_resource).
 
--export([]).
+-export([new/3,
+         lock/2,
+         download/4,
+         needs_update/2,
+         make_vsn/2]).
 
--export_type([resource/0
-             ,type/0
-             ,location/0
-              class="p">,ref/0]).
+-export_type([source/0,
+              type/0,
+              location/0,
+              ref/0]).
 
--type resource() :: {type(), location(), ref()}.
+-include("rebar.hrl").
+
+-type source() :: {type(), location(), ref()} | {type(), location(), ref(), binary()}.
 -type type() :: atom().
 -type location() :: string().
 -type ref() :: any().
 
 -callback lock(file:filename_all(), tuple()) ->
-    
rebar_resource:resource().
+    source().
 -callback download(file:filename_all(), tuple(), rebar_state:t()) ->
     {tarball, file:filename_all()} | {ok, any()} | {error, any()}.
 -callback needs_update(file:filename_all(), tuple()) ->
     boolean().
 -callback make_vsn(file:filename_all()) ->
     {plain, string()} | {error, string()}.
+
+-spec new(type(), module(), term()) -> rebar_resource_v2:resource().
+new(Type, Module, State) ->
+    #resource{type=Type,
+              module=Module,
+              state=State,
+              implementation=?MODULE}.
+
+lock(Module, AppInfo) ->
+    Module:lock(rebar_app_info:dir(AppInfo), rebar_app_info:source(AppInfo)).
+
+download(Module, TmpDir, AppInfo, State) ->
+    case Module:download(TmpDir, rebar_app_info:source(AppInfo), State) of
+        {ok, _} ->
+            ok;
+        Error ->
+            Error
+    end.
+
+needs_update(Module, AppInfo) ->
+    Module:needs_update(rebar_app_info:dir(AppInfo), rebar_app_info:source(AppInfo)).
+
+make_vsn(Module, AppInfo) ->
+    Module:make_vsn(rebar_app_info:dir(AppInfo)).

src/rebar_resource_v2.erl

@@ -0,0 +1,147 @@
+%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*-
+%% ex: ts=4 sw=4 et
+-module(rebar_resource_v2).
+
+-export([new/3,
+         find_resource_state/2,
+         format_source/1,
+         lock/2,
+         download/3,
+         needs_update/2,
+         make_vsn/3,
+         format_error/1]).
+
+-export_type([resource/0,
+              source/0,
+              type/0,
+              location/0,
+              ref/0,
+              resource_state/0]).
+
+-include("rebar.hrl").
+-include_lib("providers/include/providers.hrl").
+
+-type resource() :: #resource{}.
+-type source() :: {type(), location(), ref()} | {type(), location(), ref(), binary()}.
+-type type() :: atom().
+-type location() :: string().
+-type ref() :: any().
+-type resource_state() :: term().
+
+-callback init(type(), rebar_state:t()) -> {ok, resource()}.
+-callback lock(rebar_app_info:t(), resource_state()) -> source().
+-callback download(file:filename_all(), rebar_app_info:t(), resource_state(), rebar_state:t()) ->
+    ok | {error, any()}.
+-callback needs_update(rebar_app_info:t(), resource_state()) -> boolean().
+-callback make_vsn(rebar_app_info:t(), resource_state()) ->
+    {plain, string()} | {error, string()}.
+
+-spec new(type(), module(), term()) -> resource().
+new(Type, Module, State) ->
+    #resource{type=Type,
+              module=Module,
+              state=State,
+              implementation=?MODULE}.
+
+-spec find_resource(type(), [resource()]) -> {ok, resource()} | {error, not_found}.
+find_resource(Type, Resources) ->
+    case ec_lists:find(fun(#resource{type=T}) -> T =:= Type end, Resources) of
+        error when is_atom(Type) ->
+            case code:which(Type) of
+                non_existing ->
+                    {error, not_found};
+                _ ->
+                    {ok, rebar_resource:new(Type, Type, #{})}
+            end;
+        error ->
+            {error, not_found};
+        {ok, Resource} ->
+            {ok, Resource}
+    end.
+
+find_resource_state(Type, Resources) ->
+    case lists:keyfind(Type, #resource.type, Resources) of
+        false ->            
+            {error, not_found};
+        #resource{state=State} ->
+            State
+    end.
+
+format_source({pkg, Name, Vsn, _Hash, _}) -> {pkg, Name, Vsn};
+format_source(Source) -> Source.
+
+lock(AppInfo, State) ->
+    resource_run(lock, rebar_app_info:source(AppInfo), [AppInfo], State).
+
+resource_run(Function, Source, Args, State) ->
+    Resources = rebar_state:resources(State),
+    case get_resource_type(Source, Resources) of
+        {ok, #resource{type=_,
+                       module=Module,
+                       state=ResourceState,
+                       implementation=?MODULE}} ->
+            erlang:apply(Module, Function, Args++[ResourceState]);
+        {ok, #resource{type=_,
+                       module=Module,
+                       state=_,
+                       implementation=rebar_resource}} ->
+            erlang:apply(rebar_resource, Function, [Module | Args])
+    end.
+
+download(TmpDir, AppInfo, State) ->
+    resource_run(download, rebar_app_info:source(AppInfo), [TmpDir, AppInfo, State], State).
+
+needs_update(AppInfo, State) ->
+    resource_run(needs_update, rebar_app_info:source(AppInfo), [AppInfo], State).
+
+%% this is a special case since it is used for project apps as well, not just deps
+make_vsn(AppInfo, VcsType, State) ->
+    Resources = rebar_state:resources(State),
+    case is_resource_type(VcsType, Resources) of
+        true ->
+            case find_resource(VcsType, Resources) of
+                {ok, #resource{type=_,
+                               module=Module,
+                               state=ResourceState,
+                               implementation=?MODULE}} ->
+                    Module:make_vsn(AppInfo, ResourceState);
+                {ok, #resource{type=_,
+                               module=Module,
+                               state=_,
+                               implementation=rebar_resource}} ->
+                    rebar_resource:make_vsn(Module, AppInfo)
+            end;
+        false ->
+            unknown
+    end.
+
+format_error({no_resource, Location, Type}) ->
+    io_lib:format("Cannot handle dependency ~ts.~n"
+                  "     No module found for resource type ~p.", [Location, Type]);
+format_error({no_resource, Source}) ->
+    io_lib:format("Cannot handle dependency ~ts.~n"
+                  "     No module found for unknown resource type.", [Source]).
+
+is_resource_type(Type, Resources) ->
+    lists:any(fun(#resource{type=T}) -> T =:= Type end, Resources).
+
+-spec get_resource_type(term(), [resource()]) -> {ok, resource()}.
+get_resource_type({Type, Location}, Resources) ->
+    get_resource(Type, Location, Resources);
+get_resource_type({Type, Location, _}, Resources) ->
+    get_resource(Type, Location, Resources);
+get_resource_type({Type, _, _, Location}, Resources) ->
+    get_resource(Type, Location, Resources);
+get_resource_type(Location={Type, _, _, _, _}, Resources) ->
+    get_resource(Type, Location, Resources);
+get_resource_type(Source, _) ->
+    throw(?PRV_ERROR({no_resource, Source})).
+
+-spec get_resource(type(), term(), [resource()]) -> {ok, resource()}.
+get_resource(Type, Location, Resources) ->
+    case find_resource(Type, Resources) of
+        {error, not_found} ->
+            throw(?PRV_ERROR({no_resource, Location, Type}));
+        {ok, Resource} ->
+            {ok, Resource}
+    end.

src/rebar_state.erl

@@ -38,6 +38,12 @@
 
          to_list/1,
 
+         compilers/1, compilers/2,
+         prepend_compilers/2, append_compilers/2,
+
+         project_builders/1, add_project_builder/3,
+
+         create_resources/2, set_resources/2,
          resources/1, resources/2, add_resource/2,
          providers/1, providers/2, add_provider/2,
          allow_provider_overrides/1, allow_provider_overrides/2
@@ -65,6 +71,8 @@
                   all_plugin_deps     = []          :: [rebar_app_info:t()],
                   all_deps            = []          :: [rebar_app_info:t()],
 
+                  compilers           = []          :: [{compiler_type(), extension(), extension(), compile_fun()}],
+                  project_builders    = []          :: [{rebar_app_info:project_type(), module()}],
                   resources           = [],
                   providers           = [],
                   allow_provider_overrides = false  :: boolean()}).
@@ -73,28 +81,30 @@
 
 -type t() :: #state_t{}.
 
+-type compiler_type() :: atom().
+-type extension() :: string().
+-type compile_fun() :: fun(([file:filename()], rebar_app_info:t(), list()) -> ok).
+
 -spec new() -> t().
 new() ->
-    BaseState = base_state(),
+    BaseState = base_state(dict:new()),
     BaseState#state_t{dir = rebar_dir:get_cwd()}.
 
 -spec new(list()) -> t().
 new(Config) when is_list(Config) ->
-    BaseState = base_state(),
     Opts = base_opts(Config),
-    BaseState#state_t { dir = rebar_dir:get_cwd(),
-                        default = Opts,
-                        opts = Opts }.
+    BaseState = base_state(Opts),
+    BaseState#state_t{dir=rebar_dir:get_cwd(),
+                      default=Opts}.
 
 -spec new(t() | atom(), list()) -> t().
 new(Profile, Config) when is_atom(Profile)
                         , is_list(Config) ->
-    BaseState = base_state(),
     Opts = base_opts(Config),
-    BaseState#state_t { dir = rebar_dir:get_cwd(),
-                        current_profiles = [Profile],
-                        default = Opts,
-                        opts = Opts };
+    BaseState = base_state(Opts),
+    BaseState#state_t{dir = rebar_dir:get_cwd(),
+                      current_profiles = [Profile],
+                      default = Opts};
 new(ParentState=#state_t{}, Config) ->
     %% Load terms from rebar.config, if it exists
     Dir = rebar_dir:get_cwd(),
@@ -129,20 +139,15 @@ deps_from_config(Dir, Config) ->
             [{{locks, default}, D}, {{deps, default}, Deps}]
     end.
 
-base_state() ->
-    case application:get_env(rebar, resources) of
-        undefined ->
-            Resources = [];
-        {ok, Resources} ->
-            Resources
-    end,
-    #state_t{resources=Resources}.
+base_state(Opts) ->
+    #state_t{opts=Opts}.
 
 base_opts(Config) ->
     Deps = proplists:get_value(deps, Config, []),
     Plugins = proplists:get_value(plugins, Config, []),
     ProjectPlugins = proplists:get_value(project_plugins, Config, []),
-    Terms = [{{deps, default}, Deps}, {{plugins, default}, Plugins}, {{project_plugins, default}, ProjectPlugins} | Config],
+    Terms = [{{deps, default}, Deps}, {{plugins, default}, Plugins},
+             {{project_plugins, default}, ProjectPlugins} | Config],
     true = rebar_config:verify_config_format(Terms),
     dict:from_list(Terms).
 
@@ -359,18 +364,80 @@ namespace(#state_t{namespace=Namespace}) ->
 namespace(State=#state_t{}, Namespace) ->
     State#state_t{namespace=Namespace}.
 
--spec resources(t()) -> [{rebar_resource:type(), module()}].
+-spec resources(t()) -> [{rebar_resource_v2:type(), module()}].
 resources(#state_t{resources=Resources}) ->
     Resources.
 
--spec resources(t(), [{rebar_resource:type(), module()}]) -> t().
-resources(State, NewResources) ->
-    State#state_t{resources=NewResources}.
+-spec set_resources(t(), [{rebar_resource_v2:type(), module()}]) -> t().
+set_resources(State, Resources) ->
+    State#state_t{resources=Resources}.
 
--spec add_resource(t(), {rebar_resource:type(), module()}) -> t().
-add_resource(State=#state_t{resources=Resources}, Resource) ->
+-spec resources(t(), [{rebar_resource_v2:type(), module()}]) -> t().
+resources(State, NewResources) ->
+    lists:foldl(fun(Resource, StateAcc) ->
+                        add_resource(StateAcc, Resource)
+                end, State, NewResources).
+
+-spec add_resource(t(), {rebar_resource_v2:type(), module()}) -> t().
+add_resource(State=#state_t{resources=Resources}, {ResourceType, ResourceModule}) ->
+    _ = code:ensure_loaded(ResourceModule),
+    Resource = case erlang:function_exported(ResourceModule, init, 2) of
+                   true ->
+                       case ResourceModule:init(ResourceType, State) of
+                           {ok, R=#resource{}} ->
+                               R;
+                           _ ->
+                               %% init didn't return a resource
+                               %% must be an old resource
+                               warn_old_resource(ResourceModule),
+                               rebar_resource:new(ResourceType,
+                                                  ResourceModule,
+                                                  #{})
+                       end;
+                   false ->
+                       %% no init, must be initial implementation
+                       warn_old_resource(ResourceModule),
+                       rebar_resource:new(ResourceType,
+                                          ResourceModule,
+                                          #{})
+               end,
     State#state_t{resources=[Resource | Resources]}.
 
+warn_old_resource(ResourceModule) ->
+    ?WARN("Using custom resource ~s that implements a deprecated api. "
+          "It should be upgraded to rebar_resource_v2.", [ResourceModule]).
+
+compilers(#state_t{compilers=Compilers}) ->
+    Compilers.
+
+prepend_compilers(State=#state_t{compilers=Compilers}, NewCompilers) ->
+    State#state_t{compilers=NewCompilers++Compilers}.
+
+append_compilers(State=#state_t{compilers=Compilers}, NewCompilers) ->
+    State#state_t{compilers=Compilers++NewCompilers}.
+
+compilers(State, Compilers) ->
+    State#state_t{compilers=Compilers}.
+
+project_builders(#state_t{project_builders=ProjectBuilders}) ->
+    ProjectBuilders.
+
+add_project_builder(State=#state_t{project_builders=ProjectBuilders}, Type, Module) ->
+    _ = code:ensure_loaded(Module),
+    case erlang:function_exported(Module, build, 1) of
+        true ->
+            State#state_t{project_builders=[{Type, Module} | ProjectBuilders]};
+        false ->
+            ?WARN("Unable to add project builder for type ~s, required function ~s:build/1 not found.",
+                  [Type, Module]),
+            State
+    end.
+
+create_resources(Resources, State) ->
+    lists:foldl(fun(R, StateAcc) ->
+                        add_resource(StateAcc, R)
+                end, State, Resources).
+
 providers(#state_t{providers=Providers}) ->
     Providers.
 

src/rebar_string.erl

@@ -1,7 +1,7 @@
 %%% @doc Compatibility module for string functionality
 %%% for pre- and post-unicode support.
 -module(rebar_string).
--export([join/2, lexemes/2, trim/3, uppercase/1, lowercase/1, chr/2]).
+-export([join/2, split/2, lexemes/2, trim/3, uppercase/1, lowercase/1, chr/2]).
 
 -ifdef(unicode_str).
 
@@ -15,6 +15,7 @@ join([], Sep) when is_list(Sep) ->
 join([H|T], Sep) ->
     H ++ lists:append([Sep ++ X || X <- T]).
 
+split(Str, SearchPattern) -> string:split(Str, SearchPattern).
 lexemes(Str, SepList) -> string:lexemes(Str, SepList).
 trim(Str, Direction, Cluster=[_]) -> string:trim(Str, Direction, Cluster).
 uppercase(Str) -> string:uppercase(Str).
@@ -27,6 +28,8 @@ chr([], _C, _I) -> 0.
 -else.
 
 join(Strings, Separator) -> string:join(Strings, Separator).
+split(Str, SearchPattern) when is_list(Str) -> string:split(Str, SearchPattern);
+split(Str, SearchPattern) when is_binary(Str) -> binary:split(Str, SearchPattern).
 lexemes(Str, SepList) -> string:tokens(Str, SepList).
 trim(Str, Direction, [Char]) ->
     Dir = case Direction of

src/rebar_utils.erl

@@ -37,8 +37,9 @@
          escript_foldl/3,
          find_files/2,
          find_files/3,
+         find_files_in_dirs/3,
+         find_source/3,
          beam_to_mod/1,
-         beam_to_mod/2,
          erl_to_mod/1,
          beams/1,
          find_executable/1,
@@ -72,15 +73,17 @@
          info_useless/2,
          list_dir/1,
          user_agent/0,
-         reread_config/1,
+         reread_config/1, reread_config/2,
          get_proxy_auth/0,
-         is_list_of_strings/1]).
+         is_list_of_strings/1,
+         ssl_opts/1]).
 
 
 %% for internal use only
 -export([otp_release/0]).
 
 -include("rebar.hrl").
+-include_lib("public_key/include/OTP-PUB-KEY.hrl").
 
 -define(ONE_LEVEL_INDENT, "     ").
 -define(APP_NAME_INDEX, 2).
@@ -204,6 +207,12 @@ sh(Command0, Options0) ->
 find_files(Dir, Regex) ->
     find_files(Dir, Regex, true).
 
+find_files_in_dirs([], _Regex, _Recursive) ->
+    [];
+find_files_in_dirs([Dir | T], Regex, Recursive) ->
+    find_files(Dir, Regex, Recursive) ++ find_files_in_dirs(T, Regex, Recursive).
+
+
 find_files(Dir, Regex, Recursive) ->
     filelib:fold_files(Dir, Regex, Recursive,
                        fun(F, Acc) -> [F | Acc] end, []).
@@ -436,6 +445,18 @@ user_agent() ->
     ?FMT("Rebar/~ts (OTP/~ts)", [Vsn, otp_release()]).
 
 reread_config(ConfigList) ->
+    %% Default to not re-configuring the logger for now;
+    %% this can leak logs in CT redirection when setting up hooks
+    %% for example. If we want to turn it on by default, we may
+    %% want to disable it in CT at the same time or figure out a
+    %% way to silence it.
+    %% The same pattern may apply to other tasks, so let's enable
+    %% case-by-case.
+    reread_config(ConfigList, []).
+
+reread_config(ConfigList, Opts) ->
+    UpdateLoggerConfig = erlang:function_exported(logger, module_info, 0) andalso
+                         proplists:get_value(update_logger, Opts, false),
     %% NB: we attempt to mimic -config here, which survives app reload,
     %% hence {persistent, true}.
     SetEnv = case version_tuple(?MODULE:otp_release()) of
@@ -445,15 +466,52 @@ reread_config(ConfigList) ->
             fun (App, Key, Val) -> application:set_env(App, Key, Val, [{persistent, true}]) end
     end,
     try
+        Res =
         [SetEnv(Application, Key, Val)
         || Config <- ConfigList,
            {Application, Items} <- Config,
-           {Key, Val} <- Items]
+           {Key, Val} <- Items],
+        case UpdateLoggerConfig of
+            true -> reread_logger_config();
+            false -> ok
+        end,
+        Res
     catch _:_ ->
             ?ERROR("The configuration file submitted could not be read "
                   "and will be ignored.", [])
     end.
 
+%% @private since the kernel app is already booted, re-reading its config
+%% requires doing some magic to dynamically patch running handlers to
+%% deal with the current value.
+reread_logger_config() ->
+    KernelCfg = application:get_all_env(kernel),
+    LogCfg = proplists:get_value(logger, KernelCfg),
+    case LogCfg of
+        undefined ->
+            ok;
+        _ ->
+            %% Extract and apply settings related to primary configuration
+            %% -- primary config is used for settings shared across handlers
+            LogLvlPrimary = proplists:get_value(logger_info, KernelCfg, all),
+            {FilterDefault, Filters} =
+              case lists:keyfind(filters, 1, KernelCfg) of
+                  false -> {log, []};
+                  {filters, FoundDef, FoundFilter} -> {FoundDef, FoundFilter}
+              end,
+            Primary = #{level => LogLvlPrimary,
+                        filter_default => FilterDefault,
+                        filters => Filters},
+            %% Load the correct handlers based on their individual config.
+            [case Id of
+                 default -> logger:update_handler_config(Id, Cfg);
+                 _ -> logger:add_handler(Id, Mod, Cfg)
+             end || {handler, Id, Mod, Cfg} <- LogCfg],
+            logger:set_primary_config(Primary),
+            ok
+    end.
+
+
 %% @doc Given env. variable `FOO' we want to expand all references to
 %% it in `InStr'. References can have two forms: `$FOO' and `${FOO}'
 %% The end of form `$FOO' is delimited with whitespace or EOL
@@ -620,10 +678,6 @@ sh_loop(Port, Fun, Acc) ->
             end
     end.
 
-beam_to_mod(Dir, Filename) ->
-    [Dir | Rest] = filename:split(Filename),
-    list_to_atom(filename:basename(rebar_string:join(Rest, "."), ".beam")).
-
 beam_to_mod(Filename) ->
     list_to_atom(filename:basename(Filename, ".beam")).
 
@@ -661,12 +715,21 @@ escript_foldl(Fun, Acc, File) ->
             Error
     end.
 
-vcs_vsn(Vcs, Dir, Resources) ->
-    case vcs_vsn_cmd(Vcs, Dir, Resources) of
+%% TODO: this is just for rebar3_hex and maybe other plugins
+%% but eventually it should be dropped
+vcs_vsn(OriginalVsn, Dir, Resources) when is_list(Dir) ,
+                                          is_list(Resources) ->
+    ?WARN("Using deprecated rebar_utils:vcs_vsn/3. Please upgrade your plugins.", []),
+    FakeState = rebar_state:new(),
+    {ok, AppInfo} = rebar_app_info:new(fake, OriginalVsn, Dir),
+    vcs_vsn(AppInfo, OriginalVsn,
+            rebar_state:set_resources(FakeState, Resources));
+vcs_vsn(AppInfo, Vcs, State) ->
+    case vcs_vsn_cmd(AppInfo, Vcs, State) of
         {plain, VsnString} ->
             VsnString;
         {cmd, CmdString} ->
-            vcs_vsn_invoke(CmdString, Dir);
+            vcs_vsn_invoke(CmdString, rebar_app_info:dir(AppInfo));
         unknown ->
             ?ABORT("vcs_vsn: Unknown vsn format: ~p", [Vcs]);
         {error, Reason} ->
@@ -674,23 +737,18 @@ vcs_vsn(Vcs, Dir, Resources) ->
     end.
 
 %% Temp work around for repos like relx that use "semver"
-vcs_vsn_cmd(Vsn, _, _) when is_binary(Vsn) ->
+vcs_vsn_cmd(_, Vsn, _) when is_binary(Vsn) ->
     {plain, Vsn};
-vcs_vsn_cmd(VCS, Dir, Resources) when VCS =:= semver ; VCS =:= "semver" ->
-    vcs_vsn_cmd(git, Dir, Resources);
-vcs_vsn_cmd({cmd, _Cmd}=Custom, _, _) ->
+vcs_vsn_cmd(AppInfo, VCS, State) when VCS =:= semver ; VCS =:= "semver" ->
+    vcs_vsn_cmd(AppInfo, git, State);
+vcs_vsn_cmd(_AppInfo, {cmd, _Cmd}=Custom, _) ->
     Custom;
-vcs_vsn_cmd(VCS, Dir, Resources) when is_atom(VCS) ->
-    case find_resource_module(VCS, Resources) of
-        {ok, Module} ->
-            Module:make_vsn(Dir);
-        {error, _} ->
-            unknown
-    end;
-vcs_vsn_cmd(VCS, Dir, Resources) when is_list(VCS) ->
+vcs_vsn_cmd(AppInfo, VCS, State) when is_atom(VCS) ->
+    rebar_resource_v2:make_vsn(AppInfo, VCS, State);
+vcs_vsn_cmd(AppInfo, VCS, State) when is_list(VCS) ->
     try list_to_existing_atom(VCS) of
         AVCS ->
-            case vcs_vsn_cmd(AVCS, Dir, Resources) of
+            case vcs_vsn_cmd(AppInfo, AVCS, State) of
                 unknown -> {plain, VCS};
                 Other -> Other
             end
@@ -705,19 +763,6 @@ vcs_vsn_invoke(Cmd, Dir) ->
     {ok, VsnString} = rebar_utils:sh(Cmd, [{cd, Dir}, {use_stdout, false}]),
     rebar_string:trim(VsnString, trailing, "\n").
 
-find_resource_module(Type, Resources) ->
-    case lists:keyfind(Type, 1, Resources) of
-        false ->
-            case code:which(Type) of
-                non_existing ->
-                    {error, unknown};
-                _ ->
-                    {ok, Type}
-            end;
-        {Type, Module} ->
-            {ok, Module}
-    end.
-
 %% @doc ident to the level specified
 -spec indent(non_neg_integer()) -> iolist().
 indent(Amount) when erlang:is_integer(Amount) ->
@@ -928,3 +973,190 @@ is_list_of_strings(List) when is_list(hd(List)) ->
     true;
 is_list_of_strings(List) when is_list(List) ->
     true.
+
+%%------------------------------------------------------------------------------
+%% @doc
+%% Return the SSL options adequate for the project based on
+%% its configuration, including for validation of certs.
+%% @end
+%%------------------------------------------------------------------------------
+-spec ssl_opts(Url) -> Res when
+      Url :: string(),
+      Res :: proplists:proplist().
+ssl_opts(Url) ->
+    case get_ssl_config() of
+        ssl_verify_enabled ->
+            ssl_opts(ssl_verify_enabled, Url);
+        ssl_verify_disabled ->
+            [{verify, verify_none}]
+    end.
+
+%%------------------------------------------------------------------------------
+%% @doc
+%% Return the SSL options adequate for the project based on
+%% its configuration, including for validation of certs.
+%% @end
+%%------------------------------------------------------------------------------
+-spec ssl_opts(Enabled, Url) -> Res when
+      Enabled :: atom(),
+      Url :: string(),
+      Res :: proplists:proplist().
+ssl_opts(ssl_verify_enabled, Url) ->
+    case check_ssl_version() of
+        true ->
+            {ok, {_, _, Hostname, _, _, _}} =
+                http_uri:parse(rebar_utils:to_list(Url)),
+            VerifyFun = {fun ssl_verify_hostname:verify_fun/3,
+                         [{check_hostname, Hostname}]},
+            CACerts = certifi:cacerts(),
+            [{verify, verify_peer}, {depth, 2}, {cacerts, CACerts},
+             {partial_chain, fun partial_chain/1}, {verify_fun, VerifyFun}];
+        false ->
+            ?WARN("Insecure HTTPS request (peer verification disabled), "
+                  "please update to OTP 17.4 or later", []),
+            [{verify, verify_none}]
+    end.
+
+-spec partial_chain(Certs) -> Res when
+      Certs :: list(any()),
+      Res :: unknown_ca | {trusted_ca, any()}.
+partial_chain(Certs) ->
+    Certs1 = [{Cert, public_key:pkix_decode_cert(Cert, otp)} || Cert <- Certs],
+    CACerts = certifi:cacerts(),
+    CACerts1 = [public_key:pkix_decode_cert(Cert, otp) || Cert <- CACerts],
+    case ec_lists:find(fun({_, Cert}) ->
+                               check_cert(CACerts1, Cert)
+                       end, Certs1) of
+        {ok, Trusted} ->
+            {trusted_ca, element(1, Trusted)};
+        _ ->
+            unknown_ca
+    end.
+
+-spec extract_public_key_info(Cert) -> Res when
+      Cert :: #'OTPCertificate'{tbsCertificate::#'OTPTBSCertificate'{}},
+      Res :: any().
+extract_public_key_info(Cert) ->
+    ((Cert#'OTPCertificate'.tbsCertificate)#'OTPTBSCertificate'.subjectPublicKeyInfo).
+
+-spec check_cert(CACerts, Cert) -> Res when
+      CACerts :: list(any()),
+      Cert :: any(),
+      Res :: boolean().
+check_cert(CACerts, Cert) ->
+    lists:any(fun(CACert) ->
+                      extract_public_key_info(CACert) == extract_public_key_info(Cert)
+              end, CACerts).
+
+-spec check_ssl_version() ->
+    boolean().
+check_ssl_version() ->
+    case application:get_key(ssl, vsn) of
+        {ok, Vsn} ->
+            parse_vsn(Vsn) >= {5, 3, 6};
+        _ ->
+            false
+    end.
+
+-spec get_ssl_config() ->
+      ssl_verify_disabled | ssl_verify_enabled.
+get_ssl_config() ->
+    GlobalConfigFile = rebar_dir:global_config(),
+    Config = rebar_config:consult_file(GlobalConfigFile),
+    case proplists:get_value(ssl_verify, Config, []) of
+        false ->
+            ssl_verify_disabled;
+        _ ->
+            ssl_verify_enabled
+    end.
+
+-spec parse_vsn(Vsn) -> Res when
+      Vsn :: string(),
+      Res :: {integer(), integer(), integer()}.
+parse_vsn(Vsn) ->
+    version_pad(rebar_string:lexemes(Vsn, ".-")).
+
+-spec version_pad(list(nonempty_string())) -> Res when
+      Res :: {integer(), integer(), integer()}.
+version_pad([Major]) ->
+    {list_to_integer(Major), 0, 0};
+version_pad([Major, Minor]) ->
+    {list_to_integer(Major), list_to_integer(Minor), 0};
+version_pad([Major, Minor, Patch]) ->
+    {list_to_integer(Major), list_to_integer(Minor), list_to_integer(Patch)};
+version_pad([Major, Minor, Patch | _]) ->
+    {list_to_integer(Major), list_to_integer(Minor), list_to_integer(Patch)}.
+
+
+-ifdef(filelib_find_source).
+find_source(Filename, Dir, Rules) ->
+    filelib:find_source(Filename, Dir, Rules).
+-else.
+%% Looks for a file relative to a given directory
+
+-type find_file_rule() :: {ObjDirSuffix::string(), SrcDirSuffix::string()}.
+
+%% Looks for a source file relative to the object file name and directory
+
+-type find_source_rule() :: {ObjExtension::string(), SrcExtension::string(),
+                             [find_file_rule()]}.
+
+keep_suffix_search_rules(Rules) ->
+    [T || {_,_,_}=T <- Rules].
+
+-spec find_source(file:filename(), file:filename(), [find_source_rule()]) ->
+        {ok, file:filename()} | {error, not_found}.
+find_source(Filename, Dir, Rules) ->
+    try_suffix_rules(keep_suffix_search_rules(Rules), Filename, Dir).
+
+try_suffix_rules(Rules, Filename, Dir) ->
+    Ext = filename:extension(Filename),
+    try_suffix_rules(Rules, filename:rootname(Filename, Ext), Dir, Ext).
+
+try_suffix_rules([{Ext,Src,Rules}|Rest], Root, Dir, Ext)
+  when is_list(Src), is_list(Rules) ->
+    case try_dir_rules(add_local_search(Rules), Root ++ Src, Dir) of
+        {ok, File} -> {ok, File};
+        _Other ->
+            try_suffix_rules(Rest, Root, Dir, Ext)
+    end;
+try_suffix_rules([_|Rest], Root, Dir, Ext) ->
+    try_suffix_rules(Rest, Root, Dir, Ext);
+try_suffix_rules([], _Root, _Dir, _Ext) ->
+    {error, not_found}.
+
+%% ensuring we check the directory of the object file before any other directory
+add_local_search(Rules) ->
+    Local = {"",""},
+    [Local] ++ lists:filter(fun (X) -> X =/= Local end, Rules).
+
+try_dir_rules([{From, To}|Rest], Filename, Dir)
+  when is_list(From), is_list(To) ->
+    case try_dir_rule(Dir, Filename, From, To) of
+	{ok, File} -> {ok, File};
+	error      -> try_dir_rules(Rest, Filename, Dir)
+    end;
+try_dir_rules([], _Filename, _Dir) ->
+    {error, not_found}.
+
+try_dir_rule(Dir, Filename, From, To) ->
+    case lists:suffix(From, Dir) of
+	true ->
+	    NewDir = lists:sublist(Dir, 1, length(Dir)-length(From))++To,
+	    Src = filename:join(NewDir, Filename),
+	    case filelib:is_regular(Src) of
+		true -> {ok, Src};
+		false -> find_regular_file(filelib:wildcard(Src))
+	    end;
+	false ->
+	    error
+    end.
+
+find_regular_file([]) ->
+    error;
+find_regular_file([File|Files]) ->
+    case filelib:is_regular(File) of
+        true -> {ok, File};
+        false -> find_regular_file(Files)
+    end.
+-endif.

test/mock_git_resource.erl

@@ -27,7 +27,7 @@ mock(Opts) ->
     mock(Opts, create_app).
 
 mock(Opts, CreateType) ->
-    meck:new(?MOD, [no_link]),
+    meck:new(?MOD, [no_link, passthrough]),
     mock_lock(Opts),
     mock_update(Opts),
     mock_vsn(Opts),
@@ -46,8 +46,8 @@ unmock() ->
 mock_lock(_) ->
     meck:expect(
         ?MOD, lock,
-        fun(_AppDir, Git) ->
-            case Git of
+        fun(AppInfo, _) ->
+            case rebar_app_info:source(AppInfo) of
                 {git, Url, {tag, Ref}} -> {git, Url, {ref, Ref}};
                 {git, Url, {ref, Ref}} -> {git, Url, {ref, Ref}};
                 {git, Url} -> {git, Url, {ref, "0.0.0"}};
@@ -62,7 +62,8 @@ mock_update(Opts) ->
 %    ct:pal("TOUp: ~p", [ToUpdate]),
     meck:expect(
         ?MOD, needs_update,
-        fun(_Dir, {git, Url, _Ref}) ->
+        fun(AppInfo, _) ->
+            {git, Url, _Ref} = rebar_app_info:source(AppInfo),
             App = app(Url),
 %            ct:pal("Needed update? ~p (~p) -> ~p", [App, {Url,_Ref}, lists:member(App, ToUpdate)]),
             lists:member(App, ToUpdate)
@@ -78,7 +79,8 @@ mock_vsn(Opts) ->
     Default = proplists:get_value(default_vsn, Opts, "0.0.0"),
     meck:expect(
         ?MOD, make_vsn,
-        fun(Dir) ->
+      fun(AppInfo, _) ->
+            Dir = rebar_app_info:dir(AppInfo),
             case filelib:wildcard("*.app.src", filename:join([Dir,"src"])) of
                 [AppSrc] ->
                     {ok, App} = file:consult(AppSrc),
@@ -108,7 +110,8 @@ mock_download(Opts, CreateType) ->
     Overrides = proplists:get_value(override_vsn, Opts, []),
     meck:expect(
         ?MOD, download,
-        fun (Dir, Git, _) ->
+        fun (Dir, AppInfo, _, _) ->
+            Git = rebar_app_info:source(AppInfo),
             filelib:ensure_dir(Dir),
             {git, Url, {_, Vsn}} = normalize_git(Git, Overrides, Default),
             App = app(Url),
@@ -118,7 +121,7 @@ mock_download(Opts, CreateType) ->
                 [kernel, stdlib] ++ [element(1,D) || D  <- AppDeps]
             ),
             rebar_test_utils:create_config(Dir, [{deps, AppDeps}]++Config),
-            {ok, 'WHATEVER'}
+            ok
         end).
 
 %%%%%%%%%%%%%%%

test/mock_pkg_resource.erl

@@ -3,6 +3,8 @@
 -export([mock/0, mock/1, unmock/0]).
 -define(MOD, rebar_pkg_resource).
 
+-include("rebar.hrl").
+
 %%%%%%%%%%%%%%%%%
 %%% Interface %%%
 %%%%%%%%%%%%%%%%%
@@ -26,7 +28,7 @@ mock() -> mock([]).
     Vsn :: string(),
     Hash :: string() | undefined.
 mock(Opts) ->
-    meck:new(?MOD, [no_link]),
+    meck:new(?MOD, [no_link, passthrough]),
     mock_lock(Opts),
     mock_update(Opts),
     mock_vsn(Opts),
@@ -44,7 +46,10 @@ unmock() ->
 
 %% @doc creates values for a lock file.
 mock_lock(_) ->
-    meck:expect(?MOD, lock, fun(_AppDir, Source) -> Source end).
+    meck:expect(?MOD, lock, fun(AppInfo, _) ->
+                                {pkg, Name, Vsn, Hash, _RepoConfig} = rebar_app_info:source(AppInfo),
+                                {pkg, Name, Vsn, Hash}
+                            end).
 
 %% @doc The config passed to the `mock/2' function can specify which apps
 %% should be updated on a per-name basis: `{update, ["App1", "App3"]}'.
@@ -52,7 +57,8 @@ mock_update(Opts) ->
     ToUpdate = proplists:get_value(upgrade, Opts, []),
     meck:expect(
         ?MOD, needs_update,
-        fun(_Dir, {pkg, App, _Vsn, _Hash}) ->
+        fun(AppInfo, _) ->
+            {pkg, App, _Vsn, _Hash, _} = rebar_app_info:source(AppInfo),
             lists:member(binary_to_list(App), ToUpdate)
         end).
 
@@ -60,7 +66,7 @@ mock_update(Opts) ->
 mock_vsn(_Opts) ->
     meck:expect(
         ?MOD, make_vsn,
-        fun(_Dir) ->
+        fun(_AppInfo, _) ->
             {error, "Replacing version of type pkg not supported."}
         end).
 
@@ -77,30 +83,32 @@ mock_download(Opts) ->
     Config = proplists:get_value(config, Opts, []),
     meck:expect(
         ?MOD, download,
-        fun (Dir, {pkg, AppBin, Vsn, _}, _) ->
-            App = binary_to_list(AppBin),
+        fun (Dir, AppInfo, _, _) ->
+            {pkg, AppBin, Vsn, _, _} = rebar_app_info:source(AppInfo),
+            App = rebar_utils:to_list(AppBin),
             filelib:ensure_dir(Dir),
             AppDeps = proplists:get_value({App,Vsn}, Deps, []),
-            {ok, AppInfo} = rebar_test_utils:create_app(
-                Dir, App, binary_to_list(Vsn),
+            {ok, AppInfo1} = rebar_test_utils:create_app(
+                Dir, App, rebar_utils:to_list(Vsn),
                 [kernel, stdlib] ++ [element(1,D) || D  <- AppDeps]
             ),
             rebar_test_utils:create_config(Dir, [{deps, AppDeps}]++Config),
-            TarApp = App++"-"++binary_to_list(Vsn)++".tar",
-            Tarball = filename:join([Dir, TarApp]),
-            Contents = filename:join([Dir, "contents.tar.gz"]),
-            Files = all_files(rebar_app_info:dir(AppInfo)),
-            ok = erl_tar:create(Contents,
-                                archive_names(Dir, App, Vsn, Files),
-                                [compressed]),
-            ok = erl_tar:create(Tarball,
-                                [{"contents.tar.gz", Contents}],
-                                []),
+
+            TarApp = App++"-"++rebar_utils:to_list(Vsn)++".tar",
+
+            Metadata = #{<<"app">> => AppBin,
+                         <<"version">> => Vsn},
+
+            Files = all_files(rebar_app_info:dir(AppInfo1)),
+            {ok, {Tarball, _Checksum}} = hex_tarball:create(Metadata, archive_names(Dir, Files)),
+            Archive = filename:join([Dir, TarApp]),
+            file:write_file(Archive, Tarball),
+
             Cache = proplists:get_value(cache_dir, Opts, filename:join(Dir,"cache")),
             Cached = filename:join([Cache, TarApp]),
             filelib:ensure_dir(Cached),
-            rebar_file_utils:mv(Tarball, Cached),
-            {ok, true}
+            rebar_file_utils:mv(Archive, Cached),
+            ok
         end).
 
 %% @doc On top of the pkg resource mocking, we need to mock the package
@@ -110,16 +118,18 @@ mock_download(Opts) ->
 %% specific applications otherwise listed.
 mock_pkg_index(Opts) ->
     Deps = proplists:get_value(pkgdeps, Opts, []),
+    Repos = proplists:get_value(repos, Opts, [<<"hexpm">>]),
     Skip = proplists:get_value(not_in_index, Opts, []),
     %% Dict: {App, Vsn}: [{<<"link">>, <<>>}, {<<"deps">>, []}]
     %% Index: all apps and deps in the index
 
     Dict = find_parts(Deps, Skip),
+    to_index(Deps, Dict, Repos),
     meck:new(rebar_packages, [passthrough, no_link]),
-    meck:expect(rebar_packages, packages,
-                fun(_State) -> to_index(Deps, Dict) end),
+    meck:expect(rebar_packages, update_package,
+                fun(_, _, _State) -> ok end),
     meck:expect(rebar_packages, verify_table,
-                fun(_State) -> to_index(Deps, Dict), true end).
+                fun(_State) -> true end).
 
 %%%%%%%%%%%%%%%
 %%% Helpers %%%
@@ -128,7 +138,7 @@ mock_pkg_index(Opts) ->
 all_files(Dir) ->
     filelib:wildcard(filename:join([Dir, "**"])).
 
-archive_names(Dir, _App, _Vsn, Files) ->
+archive_names(Dir, Files) ->
     [{(F -- Dir) -- "/", F} || F <- Files].
 
 find_parts(Apps, Skip) -> find_parts(Apps, Skip, dict:new()).
@@ -143,24 +153,42 @@ find_parts([{AppName, Deps}|Rest], Skip, Acc) ->
                                 Acc),
             find_parts(Rest, Skip, AccNew)
     end.
+parse_deps(Deps) ->
+    [{maps:get(app, D, Name), {pkg, Name, Constraint, undefined}} || D=#{package := Name,
+                                                                         requirement := Constraint} <- Deps].
+
+to_index(AllDeps, Dict, Repos) ->
+    catch ets:delete(?PACKAGE_TABLE),
+    rebar_packages:new_package_table(),
 
-to_index(AllDeps, Dict) ->
-    catch ets:delete(package_index),
-    ets:new(package_index, [named_table, public]),
     dict:fold(
-      fun(K, Deps, _) ->
-              DepsList = [{DKB, {pkg, DKB, DVB, undefined}}
+      fun({N, V}, Deps, _) ->
+              DepsList = [#{package => DKB,
+                            app => DKB,
+                            requirement => DVB,
+                            source => {pkg, DKB, DVB, undefined}}
                           || {DK, DV} <- Deps,
                              DKB <- [ec_cnv:to_binary(DK)],
                              DVB <- [ec_cnv:to_binary(DV)]],
-              ets:insert(package_index, {K, DepsList, <<"checksum">>})
+              Repo = rebar_test_utils:random_element(Repos),
+              ets:insert(?PACKAGE_TABLE, #package{key={N, ec_semver:parse(V), Repo},
+                                                  dependencies=parse_deps(DepsList),
+                                                  retired=false,
+                                                  checksum = <<"checksum">>})
       end, ok, Dict),
-    ets:insert(package_index, {package_index_version, 3}),
+
     lists:foreach(fun({{Name, Vsn}, _}) ->
-                          case ets:lookup(package_index,  ec_cnv:to_binary(Name)) of
-                              [{_, Vsns}] ->
-                                  ets:insert(package_index, {ec_cnv:to_binary(Name), [ec_cnv:to_binary(Vsn) | Vsns]});
-                              _ ->
-                                  ets:insert(package_index, {ec_cnv:to_binary(Name), [ec_cnv:to_binary(Vsn)]})
+                          case lists:any(fun(R) ->
+                                                 ets:member(?PACKAGE_TABLE, {ec_cnv:to_binary(Name), ec_semver:parse(Vsn), R})
+                                         end, Repos) of
+                              false ->
+                                  Repo = rebar_test_utils:random_element(Repos),
+                                  ets:insert(?PACKAGE_TABLE, #package{key={ec_cnv:to_binary(Name), ec_semver:parse(Vsn), Repo},
+                                                                      dependencies=[],
+                                                                      retired=false,
+                                                                      checksum = <<"checksum">>});
+                              true ->
+                                  ok
                           end
                   end, AllDeps).
+

test/rebar_compile_SUITE.erl

@@ -1,69 +1,6 @@
 -module(rebar_compile_SUITE).
 
--export([suite/0,
-         init_per_suite/1,
-         end_per_suite/1,
-         init_per_testcase/2,
-         end_per_testcase/2,
-         init_per_group/2,
-         end_per_group/2,
-         all/0,
-         groups/0,
-         build_basic_app/1, paths_basic_app/1, clean_basic_app/1,
-         build_release_apps/1, paths_release_apps/1, clean_release_apps/1,
-         build_checkout_apps/1, paths_checkout_apps/1,
-         build_checkout_deps/1, paths_checkout_deps/1,
-         build_basic_srcdirs/1, paths_basic_srcdirs/1,
-         build_release_srcdirs/1, paths_release_srcdirs/1,
-         build_unbalanced_srcdirs/1, paths_unbalanced_srcdirs/1,
-         build_basic_extra_dirs/1, paths_basic_extra_dirs/1, clean_basic_extra_dirs/1,
-         build_release_extra_dirs/1, paths_release_extra_dirs/1, clean_release_extra_dirs/1,
-         build_unbalanced_extra_dirs/1, paths_unbalanced_extra_dirs/1,
-         build_extra_dirs_in_project_root/1,
-         paths_extra_dirs_in_project_root/1,
-         clean_extra_dirs_in_project_root/1,
-         recompile_when_hrl_changes/1,
-         recompile_when_included_hrl_changes/1,
-         recompile_when_opts_included_hrl_changes/1,
-         recompile_when_opts_change/1,
-         dont_recompile_when_opts_dont_change/1,
-         dont_recompile_yrl_or_xrl/1,
-         deps_in_path/1,
-         delete_beam_if_source_deleted/1,
-         checkout_priority/1,
-         highest_version_of_pkg_dep/1,
-         parse_transform_test/1,
-         erl_first_files_test/1,
-         mib_test/1,
-         umbrella_mib_first_test/1,
-         only_default_transitive_deps/1,
-         clean_all/1,
-         override_deps/1,
-         override_add_deps/1,
-         override_del_deps/1,
-         override_opts/1,
-         override_add_opts/1,
-         override_del_opts/1,
-         profile_deps/1,
-         profile_override_deps/1,
-         profile_override_add_deps/1,
-         profile_override_del_deps/1,
-         profile_override_opts/1,
-         profile_override_add_opts/1,
-         profile_override_del_opts/1,
-         deps_build_in_prod/1,
-         include_file_relative_to_working_directory/1,
-         include_file_in_src/1,
-         include_file_relative_to_working_directory_test/1,
-         include_file_in_src_test/1,
-         include_file_in_src_test_multiapp/1,
-         dont_recompile_when_erl_compiler_options_env_does_not_change/1,
-         recompile_when_erl_compiler_options_env_changes/1,
-         always_recompile_when_erl_compiler_options_set/1,
-         recompile_when_parse_transform_inline_changes/1,
-         recompile_when_parse_transform_as_opt_changes/1,
-         recursive/1,no_recursive/1,
-         regex_filter_skip/1, regex_filter_regression/1]).
+-compile(export_all).
 
 -include_lib("common_test/include/ct.hrl").
 -include_lib("eunit/include/eunit.hrl").
@@ -86,9 +23,10 @@ all() ->
      deps_in_path, checkout_priority, highest_version_of_pkg_dep,
      parse_transform_test, erl_first_files_test, mib_test,
      umbrella_mib_first_test, only_default_transitive_deps, clean_all,
-     profile_deps, deps_build_in_prod,
+     profile_deps, deps_build_in_prod, only_deps,
      override_deps, override_add_deps, override_del_deps,
      override_opts, override_add_opts, override_del_opts,
+     apply_overrides_exactly_once,
      profile_override_deps, profile_override_add_deps, profile_override_del_deps,
      profile_override_opts, profile_override_add_opts, profile_override_del_opts,
      include_file_relative_to_working_directory, include_file_in_src,
@@ -100,7 +38,8 @@ all() ->
      recursive, no_recursive,
      always_recompile_when_erl_compiler_options_set,
      dont_recompile_when_erl_compiler_options_env_does_not_change,
-     recompile_when_erl_compiler_options_env_changes].
+     recompile_when_erl_compiler_options_env_changes,
+     rebar_config_os_var].
 
 groups() ->
     [{basic_app, [], [build_basic_app, paths_basic_app, clean_basic_app]},
@@ -836,7 +775,7 @@ recompile_when_opts_change(Config) ->
 
     rebar_test_utils:create_config(AppDir, [{erl_opts, [{d, some_define}]}]),
 
-    rebar_test_utils:run_and_check(Config, [], ["compile"], {ok, [{app, Name}]}),
+    rebar_test_utils:run_and_check(Config, [{erl_opts, [{d, some_define}]}], ["compile"], {ok, [{app, Name}]}),
 
     {ok, NewFiles} = rebar_utils:list_dir(EbinDir),
     NewModTime = [filelib:last_modified(filename:join([EbinDir, F]))
@@ -893,19 +832,36 @@ dont_recompile_yrl_or_xrl(Config) ->
         "Erlang code.",
     ok = ec_file:write(Xrl, XrlBody),
 
+    Yrl = filename:join([AppDir, "src", "not_a_real_yrl_" ++ Name ++ ".yrl"]),
+    ok = filelib:ensure_dir(Yrl),
+    YrlBody = ["Nonterminals E T F.\n"
+               "Terminals '+' '*' '(' ')' number.\n"
+               "Rootsymbol E.\n"
+               "E -> E '+' T: {'$2', '$1', '$3'}.\n"
+               "E -> T : '$1'.\n"
+               "T -> T '*' F: {'$2', '$1', '$3'}.\n"
+               "T -> F : '$1'.\n"
+               "F -> '(' E ')' : '$2'.\n"
+               "F -> number : '$1'.\n"],
+    ok = ec_file:write(Yrl, YrlBody),
+
     XrlBeam = filename:join([AppDir, "ebin", filename:basename(Xrl, ".xrl") ++ ".beam"]),
+    YrlBeam = filename:join([AppDir, "ebin", filename:basename(Yrl, ".yrl") ++ ".beam"]),
 
     rebar_test_utils:run_and_check(Config, [], ["compile"], {ok, [{app, Name}]}),
 
-    ModTime = filelib:last_modified(XrlBeam),
+    XrlModTime = filelib:last_modified(XrlBeam),
+    YrlModTime = filelib:last_modified(YrlBeam),
 
     timer:sleep(1000),
 
     rebar_test_utils:run_and_check(Config, [], ["compile"], {ok, [{app, Name}]}),
 
-    NewModTime = filelib:last_modified(XrlBeam),
+    NewXrlModTime = filelib:last_modified(XrlBeam),
+    NewYrlModTime = filelib:last_modified(YrlBeam),
 
-    ?assert(ModTime == NewModTime).
+    ?assert(XrlModTime == NewXrlModTime),
+    ?assert(YrlModTime == NewYrlModTime).
 
 delete_beam_if_source_deleted(Config) ->
     AppDir = ?config(apps, Config),
@@ -1405,6 +1361,35 @@ override_opts(Config) ->
     true = lists:member(compressed, proplists:get_value(options, Mod:module_info(compile), [])),
     false = lists:member(warn_missing_spec, proplists:get_value(options, Mod:module_info(compile), [])).
 
+%% test for fix of https://github.com/erlang/rebar3/issues/1801
+%% only apply overrides once
+%% verify by having an override add the macro TEST to the dep some_dep
+%% building under `ct` will fail if the `add` is applied more than once
+apply_overrides_exactly_once(Config) ->
+    AppDir = ?config(apps, Config),
+
+    Deps = rebar_test_utils:expand_deps(git, [{"some_dep", "0.0.1", [{"other_dep", "0.0.1", []}]}]),
+    TopDeps = rebar_test_utils:top_level_deps(Deps),
+
+    {SrcDeps, _} = rebar_test_utils:flat_deps(Deps),
+    mock_git_resource:mock([{deps, SrcDeps}]),
+
+    Name = rebar_test_utils:create_random_name("app1_"),
+    Vsn = rebar_test_utils:create_random_vsn(),
+    rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]),
+
+    RebarConfig = [{deps, TopDeps},
+                   {overrides, [
+                                {add, some_dep, [
+                                             {erl_opts, [{d, 'TEST'}]}
+                                            ]}
+                               ]}],
+
+    rebar_test_utils:create_config(AppDir, RebarConfig),
+
+    rebar_test_utils:run_and_check(
+      Config, RebarConfig, ["ct", "--compile_only"], {ok, [{app, Name}, {dep, "some_dep"}], "test"}).
+
 override_add_opts(Config) ->
     AppDir = ?config(apps, Config),
 
@@ -1688,6 +1673,25 @@ profile_deps(Config) ->
         {ok, [{dep, "some_dep"},{dep, "other_dep"}]}
     ).
 
+only_deps(Config) ->
+    AppDir = ?config(apps, Config),
+
+    Name = rebar_test_utils:create_random_name("app1_"),
+    Vsn = rebar_test_utils:create_random_vsn(),
+    rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]),
+
+    Deps = rebar_test_utils:expand_deps(git, [{"some_dep", "0.0.1", [{"other_dep", "0.0.1", []}]}]),
+    TopDeps = rebar_test_utils:top_level_deps(Deps),
+    {SrcDeps, _} = rebar_test_utils:flat_deps(Deps),
+    mock_git_resource:mock([{deps, SrcDeps}]),
+
+    RConfFile = rebar_test_utils:create_config(AppDir, [{deps, TopDeps}]),
+    {ok, RConf} = file:consult(RConfFile),
+    rebar_test_utils:run_and_check(
+        Config, RConf, ["compile", "--deps_only"],
+        {ok, [{app_not_exist, Name}, {dep, "some_dep"},{dep, "other_dep"}]}
+    ).
+
 %% verify a deps prod profile is used
 %% tested by checking prod hooks run and outputs to default profile dir for dep
 %% and prod deps are installed for dep
@@ -1852,7 +1856,7 @@ include_file_in_src_test_multiapp(Config) ->
     AppDir1 = filename:join([?config(apps, Config), "lib", Name1]),
     AppDir2 = filename:join([?config(apps, Config), "lib", Name2]),
     Vsn = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_app(AppDir1, Name1, Vsn, [kernel, stdlib]),
+    rebar_test_utils:create_app(AppDir1, Name1, Vsn, [kernel, stdlib, list_to_atom(Name2)]),
     rebar_test_utils:create_app(AppDir2, Name2, Vsn, [kernel, stdlib]),
 
     Src = "-module(test).\n"
@@ -1874,7 +1878,8 @@ include_file_in_src_test_multiapp(Config) ->
     RebarConfig = [],
     rebar_test_utils:run_and_check(Config, RebarConfig,
                                    ["as", "test", "compile"],
-                                   {ok, [{app, Name1}]}).
+                                   {ok, [{app, Name1}]}),
+    ok.
 
 %% this test sets the env var, compiles, records the file last modified timestamp,
 %% recompiles and compares the file last modified timestamp to ensure it hasn't
@@ -1958,6 +1963,29 @@ recompile_when_erl_compiler_options_env_changes(Config) ->
         _     -> os:putenv("ERL_COMPILER_OPTIONS", ExistingEnv)
     end.
 
+rebar_config_os_var(Config) ->
+    AppDir = ?config(apps, Config),
+
+    Name = rebar_test_utils:create_random_name("rebar_config_os_var_"),
+    Vsn = rebar_test_utils:create_random_vsn(),
+    rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]),
+
+    rebar_test_utils:create_config(AppDir, [{erl_opts, []}]),
+
+    AltConfig = filename:join(AppDir, "test.rebar.config"),
+    file:write_file(AltConfig, "{erl_opts, [compressed]}."),
+    true = os:putenv("REBAR_CONFIG", AltConfig),
+
+    rebar_test_utils:run_and_check(Config, ["compile"], {ok, [{app, Name}]}),
+
+    Path = filename:join([AppDir, "_build", "default", "lib", Name, "ebin"]),
+    code:add_patha(Path),
+
+    Mod = list_to_atom("not_a_real_src_" ++ Name),
+
+    true = lists:member(compressed, proplists:get_value(options, Mod:module_info(compile), [])),
+    ok.
+
 %% this test sets the env var, compiles, records the file last modified
 %% timestamp, recompiles and compares the file last modified timestamp to
 %% ensure it has changed. this test should run on 18.x

test/rebar_deps_SUITE.erl

@@ -188,27 +188,27 @@ deps(flat) ->
      [],
      {ok, ["B", "C"]}};
 deps(pick_highest_left) ->
-    {[{"B", [{"C", "2", []}]},
-      {"C", "1", []}],
-     [{"C","2"}],
-     {ok, ["B", {"C", "1"}]}};
+    {[{"B", [{"C", "2.0.0", []}]},
+      {"C", "1.0.0", []}],
+     [{"C","2.0.0"}],
+     {ok, ["B", {"C", "1.0.0"}]}};
 deps(pick_highest_right) ->
-    {[{"B", "1", []},
-      {"C", [{"B", "2", []}]}],
-     [{"B","2"}],
-     {ok, [{"B","1"}, "C"]}};
+    {[{"B", "1.0.0", []},
+      {"C", [{"B", "2.0.0", []}]}],
+     [{"B","2.0.0"}],
+     {ok, [{"B","1.0.0"}, "C"]}};
 deps(pick_smallest1) ->
-    {[{"B", [{"D", "1", []}]},
-      {"C", [{"D", "2", []}]}],
-     [{"D","2"}],
+    {[{"B", [{"D", "1.0.0", []}]},
+      {"C", [{"D", "2.0.0", []}]}],
+     [{"D","2.0.0"}],
      %% we pick D1 because B < C
-     {ok, ["B","C",{"D","1"}]}};
+     {ok, ["B","C",{"D","1.0.0"}]}};
 deps(pick_smallest2) ->
-    {[{"C", [{"D", "2", []}]},
-      {"B", [{"D", "1", []}]}],
-     [{"D","2"}],
+    {[{"C", [{"D", "2.0.0", []}]},
+      {"B", [{"D", "1.0.0", []}]}],
+     [{"D","2.0.0"}],
      %% we pick D1 because B < C
-     {ok, ["B","C",{"D","1"}]}};
+     {ok, ["B","C",{"D","1.0.0"}]}};
 deps(circular1) ->
     {[{"B", [{"A", []}]}, % A is the top-level app
       {"C", []}],
@@ -222,15 +222,17 @@ deps(circular2) ->
 deps(circular_skip) ->
     %% Never spot the circular dep due to being to low in the deps tree
     %% in source deps
-    {[{"B", [{"C", "2", [{"B", []}]}]},
-      {"C", "1", [{"D",[]}]}],
-     [{"C","2"}],
-     {ok, ["B", {"C","1"}, "D"]}}.
+    {[{"B", [{"C", "2.0.0", [{"B", []}]}]},
+      {"C", "1.0.0", [{"D",[]}]}],
+     [{"C","2.0.0"}],
+     {ok, ["B", {"C","1.0.0"}, "D"]}}.
 
 setup_project(Case, Config0, Deps) ->
     DepsType = ?config(deps_type, Config0),
+    %% spread packages across 3 repos randomly
+    Repos = [<<"test-repo-1">>, <<"test-repo-2">>, <<"hexpm">>],
     Config = rebar_test_utils:init_rebar_state(
-               Config0,
+               [{repos, Repos} | Config0],
                atom_to_list(Case)++"_"++atom_to_list(DepsType)++"_"
               ),
     AppDir = ?config(apps, Config),
@@ -239,7 +241,7 @@ setup_project(Case, Config0, Deps) ->
     RebarConf = rebar_test_utils:create_config(AppDir, [{deps, TopDeps}]),
     {SrcDeps, PkgDeps} = rebar_test_utils:flat_deps(Deps),
     mock_git_resource:mock([{deps, SrcDeps}]),
-    mock_pkg_resource:mock([{pkgdeps, PkgDeps}]),
+    mock_pkg_resource:mock([{pkgdeps, PkgDeps}, {repos, Repos}]),
     [{rebarconfig, RebarConf} | Config].
 
 mock_warnings() ->
@@ -412,70 +414,62 @@ https_os_proxy_settings(_Config) ->
                  httpc:get_option(https_proxy, rebar)).
 
 semver_matching_lt(_Config) ->
-    Dep = <<"test">>,
-    Dep1 = {Dep, <<"1.0.0">>, Dep, Dep},
     MaxVsn = <<"0.2.0">>,
     Vsns = [<<"0.1.7">>, <<"0.1.9">>, <<"0.1.8">>, <<"0.2.0">>, <<"0.2.1">>],
-    ?assertEqual([{Dep, {pkg, Dep, <<"0.1.9">>, undefined}}],
-                 rebar_prv_update:cmpl_(undefined, MaxVsn, Vsns, [], Dep1,
+    ?assertEqual({ok, <<"0.1.9">>},
+                 rebar_packages:cmpl_(undefined, MaxVsn, Vsns,
                                         fun ec_semver:lt/2)).
 
 semver_matching_lte(_Config) ->
-    Dep = <<"test">>,
-    Dep1 = {Dep, <<"1.0.0">>, Dep, Dep},
     MaxVsn = <<"0.2.0">>,
     Vsns = [<<"0.1.7">>, <<"0.1.9">>, <<"0.1.8">>, <<"0.2.0">>, <<"0.2.1">>],
-    ?assertEqual([{Dep, {pkg, Dep, <<"0.2.0">>, undefined}}],
-                 rebar_prv_update:cmpl_(undefined, MaxVsn, Vsns, [], Dep1,
+    ?assertEqual({ok, <<"0.2.0">>},
+                 rebar_packages:cmpl_(undefined, MaxVsn, Vsns,
                                         fun ec_semver:lte/2)).
 
 semver_matching_gt(_Config) ->
-    Dep = <<"test">>,
-    Dep1 = {Dep, <<"1.0.0">>, Dep, Dep},
     MaxVsn = <<"0.2.0">>,
     Vsns = [<<"0.1.7">>, <<"0.1.9">>, <<"0.1.8">>, <<"0.2.0">>, <<"0.2.1">>],
-    ?assertEqual([{Dep, {pkg, Dep, <<"0.2.1">>, undefined}}],
-                 rebar_prv_update:cmp_(undefined, MaxVsn, Vsns, [], Dep1,
+    ?assertEqual({ok, <<"0.2.1">>},
+                 rebar_packages:cmp_(undefined, MaxVsn, Vsns,
                                        fun ec_semver:gt/2)).
 semver_matching_gte(_Config) ->
-    Dep = <<"test">>,
-    Dep1 = {Dep, <<"1.0.0">>, Dep, Dep},
     MaxVsn = <<"0.2.0">>,
     Vsns = [<<"0.1.7">>, <<"0.1.9">>, <<"0.1.8">>, <<"0.2.0">>],
-    ?assertEqual([{Dep, {pkg, Dep, <<"0.2.0">>, undefined}}],
-                 rebar_prv_update:cmp_(undefined, MaxVsn, Vsns, [], Dep1,
+    ?assertEqual({ok, <<"0.2.0">>},
+                 rebar_packages:cmp_(undefined, MaxVsn, Vsns, 
                                        fun ec_semver:gt/2)).
 
 valid_version(_Config) ->
-    ?assert(rebar_prv_update:valid_vsn(<<"0.1">>)),
-    ?assert(rebar_prv_update:valid_vsn(<<"0.1.0">>)),
-    ?assert(rebar_prv_update:valid_vsn(<<" 0.1.0">>)),
-    ?assert(rebar_prv_update:valid_vsn(<<"  0.1.0">>)),
-    ?assert(rebar_prv_update:valid_vsn(<<"<0.1">>)),
-    ?assert(rebar_prv_update:valid_vsn(<<"<0.1.0">>)),
-    ?assert(rebar_prv_update:valid_vsn(<<"< 0.1.0">>)),
-    ?assert(rebar_prv_update:valid_vsn(<<"<  0.1.0">>)),
-    ?assert(rebar_prv_update:valid_vsn(<<">0.1">>)),
-    ?assert(rebar_prv_update:valid_vsn(<<">0.1.0">>)),
-    ?assert(rebar_prv_update:valid_vsn(<<"> 0.1.0">>)),
-    ?assert(rebar_prv_update:valid_vsn(<<">  0.1.0">>)),
-    ?assert(rebar_prv_update:valid_vsn(<<"<=0.1">>)),
-    ?assert(rebar_prv_update:valid_vsn(<<"<=0.1.0">>)),
-    ?assert(rebar_prv_update:valid_vsn(<<"<= 0.1.0">>)),
-    ?assert(rebar_prv_update:valid_vsn(<<"<=  0.1.0">>)),
-    ?assert(rebar_prv_update:valid_vsn(<<">=0.1">>)),
-    ?assert(rebar_prv_update:valid_vsn(<<">=0.1.0">>)),
-    ?assert(rebar_prv_update:valid_vsn(<<">= 0.1.0">>)),
-    ?assert(rebar_prv_update:valid_vsn(<<">=  0.1.0">>)),
-    ?assert(rebar_prv_update:valid_vsn(<<"==0.1">>)),
-    ?assert(rebar_prv_update:valid_vsn(<<"==0.1.0">>)),
-    ?assert(rebar_prv_update:valid_vsn(<<"== 0.1.0">>)),
-    ?assert(rebar_prv_update:valid_vsn(<<"==  0.1.0">>)),
-    ?assert(rebar_prv_update:valid_vsn(<<"~>0.1">>)),
-    ?assert(rebar_prv_update:valid_vsn(<<"~>0.1.0">>)),
-    ?assert(rebar_prv_update:valid_vsn(<<"~> 0.1.0">>)),
-    ?assert(rebar_prv_update:valid_vsn(<<"~>  0.1.0">>)),
-    ?assertNot(rebar_prv_update:valid_vsn(<<"> 0.1.0 and < 0.2.0">>)),
+    ?assert(rebar_packages:valid_vsn(<<"0.1">>)),
+    ?assert(rebar_packages:valid_vsn(<<"0.1.0">>)),
+    ?assert(rebar_packages:valid_vsn(<<" 0.1.0">>)),
+    ?assert(rebar_packages:valid_vsn(<<"  0.1.0">>)),
+    ?assert(rebar_packages:valid_vsn(<<"<0.1">>)),
+    ?assert(rebar_packages:valid_vsn(<<"<0.1.0">>)),
+    ?assert(rebar_packages:valid_vsn(<<"< 0.1.0">>)),
+    ?assert(rebar_packages:valid_vsn(<<"<  0.1.0">>)),
+    ?assert(rebar_packages:valid_vsn(<<">0.1">>)),
+    ?assert(rebar_packages:valid_vsn(<<">0.1.0">>)),
+    ?assert(rebar_packages:valid_vsn(<<"> 0.1.0">>)),
+    ?assert(rebar_packages:valid_vsn(<<">  0.1.0">>)),
+    ?assert(rebar_packages:valid_vsn(<<"<=0.1">>)),
+    ?assert(rebar_packages:valid_vsn(<<"<=0.1.0">>)),
+    ?assert(rebar_packages:valid_vsn(<<"<= 0.1.0">>)),
+    ?assert(rebar_packages:valid_vsn(<<"<=  0.1.0">>)),
+    ?assert(rebar_packages:valid_vsn(<<">=0.1">>)),
+    ?assert(rebar_packages:valid_vsn(<<">=0.1.0">>)),
+    ?assert(rebar_packages:valid_vsn(<<">= 0.1.0">>)),
+    ?assert(rebar_packages:valid_vsn(<<">=  0.1.0">>)),
+    ?assert(rebar_packages:valid_vsn(<<"==0.1">>)),
+    ?assert(rebar_packages:valid_vsn(<<"==0.1.0">>)),
+    ?assert(rebar_packages:valid_vsn(<<"== 0.1.0">>)),
+    ?assert(rebar_packages:valid_vsn(<<"==  0.1.0">>)),
+    ?assert(rebar_packages:valid_vsn(<<"~>0.1">>)),
+    ?assert(rebar_packages:valid_vsn(<<"~>0.1.0">>)),
+    ?assert(rebar_packages:valid_vsn(<<"~> 0.1.0">>)),
+    ?assert(rebar_packages:valid_vsn(<<"~>  0.1.0">>)),
+    ?assertNot(rebar_packages:valid_vsn(<<"> 0.1.0 and < 0.2.0">>)),
     ok.
 
 
@@ -504,5 +498,5 @@ in_warnings(git, Warns, NameRaw, VsnRaw) ->
 in_warnings(pkg, Warns, NameRaw, VsnRaw) ->
     Name = iolist_to_binary(NameRaw),
     Vsn = iolist_to_binary(VsnRaw),
-    1 =< length([1 || {_, [AppName, {pkg, _, AppVsn, _}]} <- Warns,
+    1 =< length([1 || {_, [AppName, {pkg, _, AppVsn}]} <- Warns,
                       AppName =:= Name, AppVsn =:= Vsn]).

test/rebar_eunit_SUITE.erl

@@ -605,4 +605,4 @@ alternate_test_regex(Config) ->
 
     Set = {ok, [{application, basic_app},
                 {module, basic_app_tests}]},
-    Set = rebar_prv_eunit:prepare_tests(S).
+    Set = rebar_prv_eunit:prepare_tests(S).

test/rebar_hooks_SUITE.erl

@@ -1,20 +1,6 @@
 -module(rebar_hooks_SUITE).
 
--export([suite/0,
-         init_per_suite/1,
-         end_per_suite/1,
-         init_per_testcase/2,
-         end_per_testcase/2,
-         all/0,
-         build_and_clean_app/1,
-         escriptize_artifacts/1,
-         run_hooks_once/1,
-         run_hooks_once_profiles/1,
-         run_hooks_for_plugins/1,
-         eunit_app_hooks/1,
-         deps_hook_namespace/1,
-         bare_compile_hooks_default_ns/1,
-         deps_clean_hook_namespace/1]).
+-compile(export_all).
 
 -include_lib("common_test/include/ct.hrl").
 -include_lib("eunit/include/eunit.hrl").
@@ -38,7 +24,8 @@ end_per_testcase(_, _Config) ->
 all() ->
     [build_and_clean_app, run_hooks_once, run_hooks_once_profiles,
      escriptize_artifacts, run_hooks_for_plugins, deps_hook_namespace,
-     bare_compile_hooks_default_ns, deps_clean_hook_namespace, eunit_app_hooks].
+     bare_compile_hooks_default_ns, deps_clean_hook_namespace, eunit_app_hooks,
+     sub_app_hooks, root_hooks].
 
 %% Test post provider hook cleans compiled project app, leaving it invalid
 build_and_clean_app(Config) ->
@@ -97,7 +84,7 @@ run_hooks_once(Config) ->
 
     Name = rebar_test_utils:create_random_name("app1_"),
     Vsn = rebar_test_utils:create_random_vsn(),
-    RebarConfig = [{pre_hooks, [{compile, "mkdir blah"}]}],
+    RebarConfig = [{pre_hooks, [{compile, "mkdir  $REBAR_ROOT_DIR/blah"}]}],
     rebar_test_utils:create_config(AppDir, RebarConfig),
     rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]),
     rebar_test_utils:run_and_check(Config, RebarConfig, ["compile"], {ok, [{app, Name, valid}]}).
@@ -109,7 +96,7 @@ run_hooks_once_profiles(Config) ->
 
     Name = rebar_test_utils:create_random_name("app1_"),
     Vsn = rebar_test_utils:create_random_vsn(),
-    RebarConfig = [{profiles, [{hooks, [{pre_hooks, [{compile, "mkdir blah"}]}]}]}],
+    RebarConfig = [{profiles, [{hooks, [{pre_hooks, [{compile, "mkdir $REBAR_ROOT_DIR/blah"}]}]}]}],
     rebar_test_utils:create_config(AppDir, RebarConfig),
     rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]),
     rebar_test_utils:run_and_check(Config, RebarConfig, ["as", "hooks", "compile"], {ok, [{app, Name, valid}]}).
@@ -215,3 +202,45 @@ run_hooks_for_plugins(Config) ->
     rebar_test_utils:run_and_check(Config, RConf, ["compile"], {ok, [{app, Name, valid},
                                                                      {plugin, PluginName},
                                                                      {file, filename:join([AppDir, "_build", "default", "plugins", PluginName, "randomfile"])}]}).
+
+%% test that a subapp of a project keeps its hooks
+sub_app_hooks(Config) ->
+    AppDir = ?config(apps, Config),
+
+    Name = rebar_test_utils:create_random_name("sub_app1_"),
+    Vsn = rebar_test_utils:create_random_vsn(),
+
+    SubAppsDir = filename:join([AppDir, "apps", Name]),
+
+    rebar_test_utils:create_app(SubAppsDir, Name, Vsn, [kernel, stdlib]),
+    rebar_test_utils:create_config(SubAppsDir, [{provider_hooks, [{post, [{compile, clean}]}]}]),
+
+    RConfFile = rebar_test_utils:create_config(AppDir, []),
+    {ok, RConf} = file:consult(RConfFile),
+
+    %% Build with deps.
+    rebar_test_utils:run_and_check(
+      Config, RConf, ["compile"],
+      {ok, [{app, Name, invalid}]}
+     ).
+
+%% test that hooks at the top level don't run in the subapps
+root_hooks(Config) ->
+    AppDir = ?config(apps, Config),
+
+    Name = rebar_test_utils:create_random_name("sub_app1_"),
+    Vsn = rebar_test_utils:create_random_vsn(),
+
+    SubAppsDir = filename:join([AppDir, "apps", Name]),
+
+    rebar_test_utils:create_app(SubAppsDir, Name, Vsn, [kernel, stdlib]),
+    rebar_test_utils:create_config(SubAppsDir, [{provider_hooks, [{post, [{compile, clean}]}]}]),
+
+    RConfFile = rebar_test_utils:create_config(AppDir, [{pre_hooks, [{compile, "mkdir $REBAR_ROOT_DIR/blah"}]}]),
+    {ok, RConf} = file:consult(RConfFile),
+
+    %% Build with deps.
+    rebar_test_utils:run_and_check(
+      Config, RConf, ["compile"],
+      {ok, [{app, Name, invalid}]}
+     ).

test/rebar_install_deps_SUITE.erl

@@ -17,7 +17,7 @@ groups() ->
      {mixed, [], [
         m_flat1, m_flat2, m_circular1, m_circular2,
         m_pick_source1, m_pick_source2, m_pick_source3,
-        m_pick_source4, m_pick_source5, m_source_to_pkg,
+        m_pick_source4, m_pick_source5, m_pick_source6, m_source_to_pkg,
         m_pkg_level1, m_pkg_level2, m_pkg_level3, m_pkg_level3_alpha_order
      ]}
     ].
@@ -93,6 +93,8 @@ format_expected_mdeps(Deps) ->
              [{dep, N}, {lock, src, N, "0.0.0"}]
     end || Dep <- Deps]).
 
+format_expected_mixed_warnings(none) ->
+    none;
 format_expected_mixed_warnings(Warnings) ->
     [case W of
         {N, Vsn} when hd(N) >= $a, hd(N) =< $z -> {pkg, rebar_string:uppercase(N), Vsn};
@@ -121,27 +123,27 @@ deps(flat) ->
      [],
      {ok, ["B", "C"]}};
 deps(pick_highest_left) ->
-    {[{"B", [{"C", "2", []}]},
-      {"C", "1", []}],
-     [{"C","2"}],
-     {ok, ["B", {"C", "1"}]}};
+    {[{"B", [{"C", "2.0.0", []}]},
+      {"C", "1.0.0", []}],
+     [{"C","2.0.0"}],
+     {ok, ["B", {"C", "1.0.0"}]}};
 deps(pick_highest_right) ->
-    {[{"B", "1", []},
-      {"C", [{"B", "2", []}]}],
-     [{"B","2"}],
-     {ok, [{"B","1"}, "C"]}};
+    {[{"B", "1.0.0", []},
+      {"C", [{"B", "2.0.0", []}]}],
+     [{"B","2.0.0"}],
+     {ok, [{"B","1.0.0"}, "C"]}};
 deps(pick_smallest1) ->
-    {[{"B", [{"D", "1", []}]},
-      {"C", [{"D", "2", []}]}],
-     [{"D","2"}],
+    {[{"B", [{"D", "1.0.0", []}]},
+      {"C", [{"D", "2.0.0", []}]}],
+     [{"D","2.0.0"}],
      %% we pick D1 because B < C
-     {ok, ["B","C",{"D","1"}]}};
+     {ok, ["B","C",{"D","1.0.0"}]}};
 deps(pick_smallest2) ->
-    {[{"C", [{"D", "2", []}]},
-      {"B", [{"D", "1", []}]}],
-     [{"D","2"}],
+    {[{"C", [{"D", "2.0.0", []}]},
+      {"B", [{"D", "1.0.0", []}]}],
+     [{"D","2.0.0"}],
      %% we pick D1 because B < C
-     {ok, ["B","C",{"D","1"}]}};
+     {ok, ["B","C",{"D","1.0.0"}]}};
 deps(circular1) ->
     {[{"B", [{"A", []}]}, % A is the top-level app
       {"C", []}],
@@ -155,14 +157,14 @@ deps(circular2) ->
 deps(circular_skip) ->
     %% Never spot the circular dep due to being to low in the deps tree
     %% in source deps
-    {[{"B", [{"C", "2", [{"B", []}]}]},
-      {"C", "1", [{"D",[]}]}],
-     [{"C","2"}],
-     {ok, ["B", {"C","1"}, "D"]}};
+    {[{"B", [{"C", "2.0.0", [{"B", []}]}]},
+      {"C", "1.0.0", [{"D",[]}]}],
+     [{"C","2.0.0"}],
+     {ok, ["B", {"C","1.0.0"}, "D"]}};
 deps(fail_conflict) ->
-    {[{"B", [{"C", "2", []}]},
-      {"C", "1", []}],
-     [{"C","2"}],
+    {[{"B", [{"C", "2.0.0", []}]},
+      {"C", "1.0.0", []}],
+     [{"C","2.0.0"}],
      rebar_abort};
 deps(default_profile) ->
     {[{"B", []},
@@ -216,39 +218,44 @@ mdeps(m_pick_source3) ->
      [],
      {ok, ["B"]}};
 mdeps(m_pick_source4) ->
-    {[{"b", [{"d", "1", []}]},
-      {"C", [{"D", "1", []}]}],
-     [{"D", "1"}],
-     {ok, ["b", "C", {"d", "1"}]}};
+    {[{"b", [{"d", "1.0.0", []}]},
+      {"C", [{"D", "1.0.0", []}]}],
+     [{"D", "1.0.0"}],
+     {ok, ["b", "C", {"d", "1.0.0"}]}};
 mdeps(m_pick_source5) ->
-    {[{"B", [{"d", "1", []}]},
-      {"C", [{"D", "1", []}]}],
-     [{"D", "1"}],
-     {ok, ["B", "C", {"d", "1"}]}};
+    {[{"B", [{"d", "1.0.0", []}]},
+      {"C", [{"D", "1.0.0", []}]}],
+     [{"D", "1.0.0"}],
+     {ok, ["B", "C", {"d", "1.0.0"}]}};
+mdeps(m_pick_source6) ->
+    {[{"B", [{"D", "1.0.0", []}]},
+      {"C", [{"D", "1.0.0", []}]}],
+     none,
+     {ok, ["B", "C", {"D", "1.0.0"}]}};
 mdeps(m_source_to_pkg) ->
     {[{"B", [{"c",[{"d", []}]}]}],
      [],
      {ok, ["B", "c", "d"]}};
 mdeps(m_pkg_level1) ->
-    {[{"B", [{"D", [{"e", "2", []}]}]},
-      {"C", [{"e", "1", []}]}],
-     [{"e","2"}],
-     {ok, ["B","C","D",{"e","1"}]}};
+    {[{"B", [{"D", [{"e", "2.0.0", []}]}]},
+      {"C", [{"e", "1.0.0", []}]}],
+     [{"e","2.0.0"}],
+     {ok, ["B","C","D",{"e","1.0.0"}]}};
 mdeps(m_pkg_level2) ->
-    {[{"B", [{"e", "1", []}]},
-      {"C", [{"D", [{"e", "2", []}]}]}],
-     [{"e","2"}],
-     {ok, ["B","C","D",{"e","1"}]}};
+    {[{"B", [{"e", "1.0.0", []}]},
+      {"C", [{"D", [{"e", "2.0.0", []}]}]}],
+     [{"e","2.0.0"}],
+     {ok, ["B","C","D",{"e","1.0.0"}]}};
 mdeps(m_pkg_level3_alpha_order) ->
-    {[{"B", [{"d", [{"f", "1", []}]}]},
-      {"C", [{"E", [{"f", "2", []}]}]}],
-     [{"f","2"}],
-     {ok, ["B","C","d","E",{"f","1"}]}};
+    {[{"B", [{"d", [{"f", "1.0.0", []}]}]},
+      {"C", [{"E", [{"f", "2.0.0", []}]}]}],
+     [{"f","2.0.0"}],
+     {ok, ["B","C","d","E",{"f","1.0.0"}]}};
 mdeps(m_pkg_level3) ->
-    {[{"B", [{"d", [{"f", "1", []}]}]},
-      {"C", [{"E", [{"G", [{"f", "2", []}]}]}]}],
-     [{"f","2"}],
-     {ok, ["B","C","d","E","G",{"f","1"}]}}.
+    {[{"B", [{"d", [{"f", "1.0.0", []}]}]},
+      {"C", [{"E", [{"G", [{"f", "2.0.0", []}]}]}]}],
+     [{"f","2.0.0"}],
+     {ok, ["B","C","d","E","G",{"f","1.0.0"}]}}.
 
 setup_project(fail_conflict, Config0, Deps) ->
     DepsType = ?config(deps_type, Config0),
@@ -289,8 +296,8 @@ setup_project(nondefault_pick_highest, Config0, _) ->
     ),
     AppDir = ?config(apps, Config),
     rebar_test_utils:create_app(AppDir, "A", "0.0.0", [kernel, stdlib]),
-    DefaultDeps = rebar_test_utils:expand_deps(DepsType, [{"B", [{"C", "1", []}]}]),
-    ProfileDeps = rebar_test_utils:expand_deps(DepsType, [{"C", "2", []}]),
+    DefaultDeps = rebar_test_utils:expand_deps(DepsType, [{"B", [{"C", "1.0.0", []}]}]),
+    ProfileDeps = rebar_test_utils:expand_deps(DepsType, [{"C", "2.0.0", []}]),
     DefaultTop = rebar_test_utils:top_level_deps(DefaultDeps),
     ProfileTop = rebar_test_utils:top_level_deps(ProfileDeps),
     RebarConf = rebar_test_utils:create_config(
@@ -412,19 +419,19 @@ nondefault_pick_highest(Config) ->
     {ok, RebarConfig} = file:consult(?config(rebarconfig, Config)),
     rebar_test_utils:run_and_check(
         Config, RebarConfig, ["lock"],
-        {ok, [{dep, "B"}, {lock, "B"}, {lock, "C", "1"}, {dep, "C", "1"}], "default"}
+        {ok, [{dep, "B"}, {lock, "B"}, {lock, "C", "1.0.0"}, {dep, "C", "1.0.0"}], "default"}
     ),
     rebar_test_utils:run_and_check(
         Config, RebarConfig, ["as", "nondef", "lock"],
-        {ok, [{dep, "B"}, {lock, "B"}, {lock, "C", "1"}, {dep, "C", "2"}], "nondef"}
+        {ok, [{dep, "B"}, {lock, "B"}, {lock, "C", "1.0.0"}, {dep, "C", "2.0.0"}], "nondef"}
     ),
     rebar_test_utils:run_and_check(
         Config, RebarConfig, ["lock"],
-        {ok, [{dep, "B"}, {lock, "B"}, {dep, "C", "1"}, {lock, "C", "1"}], "default"}
+        {ok, [{dep, "B"}, {lock, "B"}, {dep, "C", "1.0.0"}, {lock, "C", "1.0.0"}], "default"}
     ),
     rebar_test_utils:run_and_check(
         Config, RebarConfig, ["as", "nondef", "lock"],
-        {ok, [{dep, "B"}, {lock, "B"}, {lock, "C", "1"}, {dep, "C", "2"}], "nondef"}
+        {ok, [{dep, "B"}, {lock, "B"}, {lock, "C", "1.0.0"}, {dep, "C", "2.0.0"}], "nondef"}
     ).
 
 m_flat1(Config) -> run(Config).
@@ -436,6 +443,7 @@ m_pick_source2(Config) -> run(Config).
 m_pick_source3(Config) -> run(Config).
 m_pick_source4(Config) -> run(Config).
 m_pick_source5(Config) -> run(Config).
+m_pick_source6(Config) -> run(Config).
 m_source_to_pkg(Config) -> run(Config).
 m_pkg_level1(Config) -> run(Config).
 m_pkg_level2(Config) -> run(Config).
@@ -466,7 +474,10 @@ check_warnings(Warns, [{Type, Name, Vsn} | Rest], mixed) ->
 check_warnings(Warns, [{Name, Vsn} | Rest], Type) ->
     ct:pal("Checking for warning ~p in ~p", [{Name,Vsn},Warns]),
     ?assert(in_warnings(Type, Warns, Name, Vsn)),
-    check_warnings(Warns, Rest, Type).
+    check_warnings(Warns, Rest, Type);
+check_warnings(Warns, none, _Type) ->
+    ct:pal("Checking that there were no warnings", []),
+    ?assert(Warns == []).
 
 in_warnings(git, Warns, NameRaw, VsnRaw) ->
     Name = iolist_to_binary(NameRaw),
@@ -475,5 +486,5 @@ in_warnings(git, Warns, NameRaw, VsnRaw) ->
 in_warnings(pkg, Warns, NameRaw, VsnRaw) ->
     Name = iolist_to_binary(NameRaw),
     Vsn = iolist_to_binary(VsnRaw),
-    1 =< length([1 || {_, [AppName, {pkg, _, AppVsn, _}]} <- Warns,
+    1 =< length([1 || {_, [AppName, {pkg, _, AppVsn}]} <- Warns,
                       AppName =:= Name, AppVsn =:= Vsn]).

test/rebar_localfs_resource.erl

@@ -2,6 +2,7 @@
 %% ex: ts=4 sw=4 et
 %%
 %% @doc A localfs custom resource (for testing purposes only)
+%% implementing the deprecated rebar_resource instead of v2
 %%
 %% ```
 %% {deps, [
@@ -13,13 +14,18 @@
 
 -behaviour(rebar_resource).
 
--export([lock/2
+-export([init/1
+        ,lock/2
         ,download/3
         ,needs_update/2
         ,make_vsn/1]).
 
 -include_lib("eunit/include/eunit.hrl").
 
+-spec init(rebar_state:t()) -> {ok, term()}.
+init(_State) ->
+    {ok, #{}}.
+
 lock(AppDir, {localfs, Path, _Ref}) ->
     lock(AppDir, {localfs, Path});
 lock(_AppDir, {localfs, Path}) ->

test/rebar_localfs_resource_v2.erl

@@ -0,0 +1,50 @@
+%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*-
+%% ex: ts=4 sw=4 et
+%%
+%% @doc A localfs custom resource (for testing purposes only)
+%%
+%% ```
+%% {deps, [
+%%     %% Application files are copied from "/path/to/app_name"
+%%     {app_name, {localfs, "/path/to/app_name", undefined}}
+%% ]}.
+%% '''
+-module(rebar_localfs_resource_v2).
+
+-behaviour(rebar_resource_v2).
+
+-export([init/2
+        ,lock/2
+        ,download/4
+        ,needs_update/2
+        ,make_vsn/2]).
+
+-include_lib("eunit/include/eunit.hrl").
+
+-spec init(atom(), rebar_state:t()) -> {ok, term()}.
+init(Type, _State) ->
+    Resource = rebar_resource_v2:new(Type, ?MODULE, #{}),
+    {ok, Resource}.
+
+lock(AppInfo, _) ->
+    case rebar_app_info:source(AppInfo) of
+        {localfs, Path, _Ref} ->
+            {localfs, Path, undefined};
+        {localfs, Path} ->
+            {localfs, Path, undefined}
+    end.
+
+needs_update(_AppInfo, _) ->
+    false.
+
+download(TmpDir, AppInfo, State, _) ->
+    download_(TmpDir, rebar_app_info:source(AppInfo), State).
+
+download_(TmpDir, {localfs, Path, _Ref}, State) ->
+    download_(TmpDir, {localfs, Path}, State);
+download_(TmpDir, {localfs, Path}, _State) ->
+    ok = rebar_file_utils:cp_r(filelib:wildcard(Path ++ "/*"), TmpDir),
+    {ok, undefined}.
+
+make_vsn(_AppInfo, _) ->
+    {plain, "undefined"}.

test/rebar_paths_SUITE.erl

@@ -0,0 +1,240 @@
+-module(rebar_paths_SUITE).
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("common_test/include/ct.hrl").
+-compile(export_all).
+
+all() ->
+    [clashing_apps,
+     check_modules,
+     set_paths,
+     misloaded_mods
+    ].
+
+%%%%%%%%%%%%%%%%%%
+%%% TEST SETUP %%%
+%%%%%%%%%%%%%%%%%%
+
+init_per_testcase(Case, Config) ->
+    BasePaths = code:get_path(),
+    %% This test checks that the right module sets get loaded; however, we must
+    %% ensure that we do not have clashes with other test suites' loaded modules,
+    %% which we cannot track. As such, we have to ensure all module names here are
+    %% unique.
+    %%
+    %% This is done by hand; if you see this test suite failing on its own, you
+    %% probably wrote a test suite that clashes!
+    Dir = filename:join([?config(priv_dir, Config), atom_to_list(?MODULE),
+                         atom_to_list(Case)]),
+    InDir = fun(Path) -> filename:join([Dir, Path]) end,
+    ADep = fake_app(<<"rp_a">>, <<"1.0.0">>, InDir("_build/default/lib/rp_a/")),
+    BDep = fake_app(<<"rp_b">>, <<"1.0.0">>, InDir("_build/default/lib/rp_b/")),
+    CDep = fake_app(<<"rp_c">>, <<"1.0.0">>, InDir("_build/default/lib/rp_c/")),
+    DDep = fake_app(<<"rp_d">>, <<"1.0.0">>, InDir("_build/default/lib/rp_d/")),
+    RelxDep = fake_app(<<"relx">>, <<"1.0.0">>, InDir("_build/default/lib/relx/")),
+
+    APlug = fake_app(<<"rp_a">>, <<"1.0.0">>,
+                     InDir("_build/default/plugins/lib/rp_a/")),
+    RelxPlug = fake_app(<<"relx">>, <<"1.1.1">>,
+                        InDir("_build/default/plugins/lib/relx")),
+    EPlug = fake_app(<<"rp_e">>, <<"1.0.0">>,
+                     InDir("_build/default/plugins/lib/rp_e/")),
+
+    S0 = rebar_state:new(),
+    S1 = rebar_state:all_deps(S0, [ADep, BDep, CDep, DDep, RelxDep]),
+    S2 = rebar_state:all_plugin_deps(S1, [APlug, RelxPlug]),
+    S3 = rebar_state:code_paths(S2, default, code:get_path()),
+    S4 = rebar_state:code_paths(
+        S3,
+        all_deps,
+        [rebar_app_info:ebin_dir(A) || A <- [ADep, BDep, CDep, DDep, RelxDep]]
+    ),
+    S5 = rebar_state:code_paths(
+        S4,
+        all_plugin_deps,
+        [rebar_app_info:ebin_dir(A) || A <- [APlug, RelxPlug, EPlug]]
+    ),
+    [{base_paths, BasePaths}, {root_dir, Dir}, {state, S5} | Config].
+
+end_per_testcase(_, Config) ->
+    %% this is deeply annoying because we interfere with rebar3's own
+    %% path handling!
+    rebar_paths:unset_paths([plugins, deps], ?config(state, Config)),
+    Config.
+
+fake_app(Name, Vsn, OutDir) ->
+    {ok, App} = rebar_app_info:new(Name, Vsn, OutDir),
+    compile_fake_appmod(App),
+    App.
+
+compile_fake_appmod(App) ->
+    OutDir = rebar_app_info:ebin_dir(App),
+    Vsn = rebar_app_info:original_vsn(App),
+    Name = rebar_app_info:name(App),
+
+    ok = filelib:ensure_dir(filename:join([OutDir, ".touch"])),
+
+    AppFile = [
+      "{application,", Name, ", "
+      " [{description,  \"some app\"}, "
+      "  {vsn, \"", Vsn, "\"}, "
+      "  {modules, [",Name,"]}, "
+      "  {registered, []}, "
+      "  {applications, [stdlib, kernel]} "
+      " ]}. "],
+
+    ok = file:write_file(filename:join([OutDir, <<Name/binary, ".app">>]), AppFile),
+
+    Mod = [{attribute, 1, module, binary_to_atom(Name, utf8)},
+           {attribute, 2, export, [{f,0}]},
+           {function,3,f,0,
+            [{clause,3, [], [],
+              [{string,3,OutDir}]
+            }]}
+          ],
+
+    {ok, _, Bin} = compile:forms(Mod),
+    ok = file:write_file(filename:join([OutDir, <<Name/binary, ".beam">>]), Bin).
+
+%%%%%%%%%%%%%
+%%% TESTS %%%
+%%%%%%%%%%%%%
+
+clashing_apps(Config) ->
+    Clashes = rebar_paths:clashing_apps([deps, plugins],
+                                        ?config(state, Config)),
+    ct:pal("Clashes: ~p", [Clashes]),
+
+    ?assertEqual([<<"relx">>, <<"rp_a">>], lists:sort(proplists:get_value(deps, Clashes))),
+    ?assertEqual([], proplists:get_value(plugins, Clashes)),
+    ok.
+
+set_paths(Config) ->
+    State = ?config(state, Config),
+    RootDir = filename:split(?config(root_dir, Config)),
+    rebar_paths:set_paths([plugins, deps], State),
+    PluginPaths = code:get_path(),
+    rebar_paths:set_paths([deps, plugins], State),
+    DepPaths = code:get_path(),
+
+    ?assertEqual(
+       RootDir ++ ["_build", "default", "plugins", "lib", "rp_a", "ebin"],
+       find_first_instance("rp_a", PluginPaths)
+    ),
+    ?assertEqual(
+       RootDir ++ ["_build", "default", "lib", "rp_b", "ebin"],
+       find_first_instance("rp_b", PluginPaths)
+    ),
+    ?assertEqual(
+       RootDir ++ ["_build", "default", "lib", "rp_c", "ebin"],
+       find_first_instance("rp_c", PluginPaths)
+    ),
+    ?assertEqual(
+       RootDir ++ ["_build", "default", "lib", "rp_d", "ebin"],
+       find_first_instance("rp_d", PluginPaths)
+    ),
+    ?assertEqual(
+       RootDir ++ ["_build", "default", "plugins", "lib",  "rp_e", "ebin"],
+       find_first_instance("rp_e", PluginPaths)
+    ),
+    ?assertEqual(
+       RootDir ++ ["_build", "default", "plugins", "lib", "relx", "ebin"],
+       find_first_instance("relx", PluginPaths)
+    ),
+
+
+    ?assertEqual(
+       RootDir ++ ["_build", "default", "lib", "rp_a", "ebin"],
+       find_first_instance("rp_a", DepPaths)
+    ),
+    ?assertEqual(
+       RootDir ++ ["_build", "default", "lib", "rp_b", "ebin"],
+       find_first_instance("rp_b", DepPaths)
+    ),
+    ?assertEqual(
+       RootDir ++ ["_build", "default", "lib", "rp_c", "ebin"],
+       find_first_instance("rp_c", DepPaths)
+    ),
+    ?assertEqual(
+       RootDir ++ ["_build", "default", "lib", "rp_d", "ebin"],
+       find_first_instance("rp_d", DepPaths)
+    ),
+    ?assertEqual(
+       RootDir ++ ["_build", "default", "plugins", "lib", "rp_e", "ebin"],
+       find_first_instance("rp_e", DepPaths)
+    ),
+    ?assertEqual(
+       RootDir ++ ["_build", "default", "lib", "relx", "ebin"],
+       find_first_instance("relx", DepPaths)
+    ),
+    ok.
+
+check_modules(Config) ->
+    State = ?config(state, Config),
+    RootDir = ?config(root_dir, Config)++"/",
+    rebar_paths:set_paths([plugins, deps], State),
+    ct:pal("code:get_path() -> ~p", [code:get_path()]),
+
+    ?assertEqual(RootDir ++ "_build/default/plugins/lib/rp_a/ebin", rp_a:f()),
+    ct:pal("~p", [catch file:list_dir(RootDir ++ "_build/default/lib/")]),
+    ct:pal("~p", [catch file:list_dir(RootDir ++ "_build/default/lib/rp_b/")]),
+    ct:pal("~p", [catch file:list_dir(RootDir ++ "_build/default/lib/rp_b/ebin")]),
+    ct:pal("~p", [catch b:module_info()]),
+    ?assertEqual(RootDir ++ "_build/default/lib/rp_b/ebin", rp_b:f()),
+    ?assertEqual(RootDir ++ "_build/default/lib/rp_c/ebin", rp_c:f()),
+    ?assertEqual(RootDir ++ "_build/default/lib/rp_d/ebin", rp_d:f()),
+    ?assertEqual(RootDir ++ "_build/default/plugins/lib/rp_e/ebin", rp_e:f()),
+    ?assertEqual(RootDir ++ "_build/default/plugins/lib/relx/ebin", relx:f()),
+    ?assertEqual(3, length(relx:module_info(exports))), % can't replace bundled
+
+    rebar_paths:set_paths([deps, plugins], State),
+    ct:pal("code:get_path() -> ~p", [code:get_path()]),
+
+    ?assertEqual(RootDir ++ "_build/default/lib/rp_a/ebin", rp_a:f()),
+    ?assertEqual(RootDir ++ "_build/default/lib/rp_b/ebin", rp_b:f()),
+    ?assertEqual(RootDir ++ "_build/default/lib/rp_c/ebin", rp_c:f()),
+    ?assertEqual(RootDir ++ "_build/default/lib/rp_d/ebin", rp_d:f()),
+    ?assertEqual(RootDir ++ "_build/default/plugins/lib/rp_e/ebin", rp_e:f()),
+    ?assertEqual(RootDir ++ "_build/default/lib/relx/ebin", relx:f()),
+    ?assertEqual(3, length(relx:module_info(exports))), % can't replace bundled
+
+    %% once again
+    rebar_paths:set_paths([plugins, deps], State),
+    ct:pal("code:get_path() -> ~p", [code:get_path()]),
+
+    ?assertEqual(RootDir ++ "_build/default/plugins/lib/rp_a/ebin", rp_a:f()),
+    ?assertEqual(RootDir ++ "_build/default/lib/rp_b/ebin", rp_b:f()),
+    ?assertEqual(RootDir ++ "_build/default/lib/rp_c/ebin", rp_c:f()),
+    ?assertEqual(RootDir ++ "_build/default/lib/rp_d/ebin", rp_d:f()),
+    ?assertEqual(RootDir ++ "_build/default/plugins/lib/rp_e/ebin", rp_e:f()),
+    ?assertEqual(RootDir ++ "_build/default/plugins/lib/relx/ebin", relx:f()),
+    ?assertEqual(3, length(relx:module_info(exports))), % can't replace bundled
+    ok.
+
+misloaded_mods(_Config) ->
+    Res = rebar_paths:misloaded_modules(
+      ["/1/2/3/4",
+       "/1/2/4",
+       "/2/1/1",
+       "/3/4/5"],
+      [{a, "/0/1/2/file.beam"},
+       {b, "/1/2/3/4/file.beam"},
+       {c, "/2/1/file.beam"},
+       {f, preloaded},
+       {d, "/3/5/7/file.beam"},
+       {e, "/3/4/5/file.beam"}]
+    ),
+    ?assertEqual([a,c,d], Res),
+    ok.
+
+%%%%%%%%%%%%%%%
+%%% HELPERS %%%
+%%%%%%%%%%%%%%%
+
+find_first_instance(Frag, []) ->
+    {not_found, Frag};
+find_first_instance(Frag, [Path|Rest]) ->
+    Frags = filename:split(Path),
+    case lists:member(Frag, Frags) of
+        true -> Frags;
+        false -> find_first_instance(Frag, Rest)
+    end.

test/rebar_pkg_SUITE.erl

@@ -4,17 +4,19 @@
 -compile(export_all).
 -include_lib("common_test/include/ct.hrl").
 -include_lib("eunit/include/eunit.hrl").
+-include("rebar.hrl").
 
--define(bad_etag, "abcdef").
--define(good_etag, "22e1d7387c9085a462340088a2a8ba67").
+-define(bad_etag, <<"abcdef">>).
+-define(good_etag, <<"22e1d7387c9085a462340088a2a8ba67">>).
+-define(badpkg_checksum, <<"A14E3718B33F8124E98004433193509EC6660F6CA03302657CAB8785751D77A0">>).
+-define(badindex_checksum, <<"7B2CBED315C89F3126B5BF553DD7FF0FB5FE94B064888DD1B095CE8BF4B6A16A">>).
 -define(bad_checksum, <<"D576B442A68C7B92BACDE1EFE9C6E54D8D6C74BDB71D8175B9D3C6EC8C7B62A7">>).
--define(good_checksum, <<"1C6CE379D191FBAB41B7905075E0BF87CBBE23C77CECE775C5A0B786B2244C35">>).
+-define(good_checksum, <<"12726BDE1F65583A0817A7E8AADCA73F03FD8CB06F01E6CD29117C4A0DA0AFCF">>).
 -define(BADPKG_ETAG, <<"BADETAG">>).
 
-all() -> [good_uncached, good_cached, badindexchk, badpkg,
-          badhash_nocache, badhash_cache,
-          bad_to_good, good_disconnect, bad_disconnect, pkgs_provider,
-          find_highest_matching].
+all() -> [good_uncached, good_cached, badpkg, badhash_nocache,
+          badindexchk, badhash_cache, bad_to_good, good_disconnect,
+          bad_disconnect, pkgs_provider, find_highest_matching].
 
 init_per_suite(Config) ->
     application:start(meck),
@@ -32,10 +34,6 @@ init_per_testcase(pkgs_provider=Name, Config) ->
     CacheDir = filename:join([CacheRoot, "hex", "com", "test", "packages"]),
     filelib:ensure_dir(filename:join([CacheDir, "registry"])),
     ok = ets:tab2file(Tid, filename:join([CacheDir, "registry"])),
-    meck:new(rebar_packages, [passthrough]),
-    meck:expect(rebar_packages, registry_dir, fun(_) -> {ok, CacheDir} end),
-    meck:expect(rebar_packages, package_dir, fun(_) -> {ok, CacheDir} end),
-    rebar_prv_update:hex_to_index(rebar_state:new()),
     Config;
 init_per_testcase(good_uncached=Name, Config0) ->
     Config = [{good_cache, false},
@@ -89,9 +87,9 @@ init_per_testcase(good_disconnect=Name, Config0) ->
               | Config0],
     Config = mock_config(Name, Config1),
     copy_to_cache(Pkg, Config),
-    meck:unload(httpc),
+    %% meck:unload(httpc),
     meck:new(httpc, [passthrough, unsticky]),
-    meck:expect(httpc, request, fun(_, _, _, _, _) -> {error, econnrefused} end),
+    meck:expect(httpc, request, fun(_, _, _, _) -> {error, econnrefused} end),
     Config;
 init_per_testcase(bad_disconnect=Name, Config0) ->
     Pkg = {<<"goodpkg">>, <<"1.0.0">>},
@@ -99,9 +97,9 @@ init_per_testcase(bad_disconnect=Name, Config0) ->
                {pkg, Pkg}
               | Config0],
     Config = mock_config(Name, Config1),
-    meck:unload(httpc),
-    meck:new(httpc, [passthrough, unsticky]),
-     class="nn">meck class="p">: class="nf">expect class="p">( class="n">httpc class="p">,  class="n">request class="p">,  class="k">fun class="p">( class="p">_ class="p">,  class="p">_ class="p">,  class="p">_ class="p">,  class="p">_ class="p">,  class="p">_ class="p">)  class="o">- class="o">>  class="p">{ class="n">error class="p">, econnrefused} end),
+    meck:expect(hex_repo, get_tarball, fun(_, _, _) ->
+                                               {error, econnrefused}
+                                       end),                
     Config;
 init_per_testcase(Name, Config0) ->
     Config = [{good_cache, false},
@@ -117,8 +115,8 @@ good_uncached(Config) ->
     Tmp = ?config(tmp_dir, Config),
     {Pkg,Vsn} = ?config(pkg, Config),
     State = ?config(state, Config),
-    ?assertEqual({ok, true},
-                 rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum}, State)),
+    ?assertEqual(ok,
+                 rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum, #{}}, State, #{}, true)),
     Cache = ?config(cache_dir, Config),
     ?assert(filelib:is_regular(filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".tar">>))).
 
@@ -130,16 +128,17 @@ good_cached(Config) ->
     CachedFile = filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".tar">>),
     ?assert(filelib:is_regular(CachedFile)),
     {ok, Content} = file:read_file(CachedFile),
-    ?assertEqual({ok, true},
-                 rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum}, State)),
+    ?assertEqual(ok,
+                 rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum, #{}}, State, #{}, true)),
     {ok, Content} = file:read_file(CachedFile).
 
+
 badindexchk(Config) ->
     Tmp = ?config(tmp_dir, Config),
     {Pkg,Vsn} = ?config(pkg, Config),
     State = ?config(state, Config),
-    ?assertMatch({bad_registry_checksum, _Path},
-                 rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum}, State)),
+    ?assertMatch({error, {rebar_pkg_resource, {bad_registry_checksum, _, _, _, _}}},
+                 rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?bad_checksum, #{}}, State, #{}, true)),
     %% The cached file is there for forensic purposes
     Cache = ?config(cache_dir, Config),
     ?assert(filelib:is_regular(filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".tar">>))).
@@ -152,8 +151,8 @@ badpkg(Config) ->
     CachePath = filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".tar">>),
     ETagPath = filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".etag">>),
     rebar_pkg_resource:store_etag_in_cache(ETagPath, ?BADPKG_ETAG),
-    ?assertMatch({bad_download, _Path},
-                 rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum}, State, false)),
+    ?assertMatch({error, {hex_tarball, {tarball, {checksum_mismatch, _, _}}}},
+                 rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?badpkg_checksum, #{}}, State, #{}, false)),
     %% The cached/etag files are there for forensic purposes
     ?assert(filelib:is_regular(ETagPath)),
     ?assert(filelib:is_regular(CachePath)).
@@ -162,8 +161,8 @@ badhash_nocache(Config) ->
     Tmp = ?config(tmp_dir, Config),
     {Pkg,Vsn} = ?config(pkg, Config),
     State = ?config(state, Config),
-    ?assertMatch({unexpected_hash, _Path, ?bad_checksum, ?good_checksum},
-                 rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?bad_checksum}, State)),
+    ?assertMatch({error, {rebar_pkg_resource, {bad_registry_checksum, _, _, _, _}}},
+                 rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?bad_checksum, #{}}, State, #{}, true)),
     %% The cached file is there for forensic purposes
     Cache = ?config(cache_dir, Config),
     ?assert(filelib:is_regular(filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".tar">>))).
@@ -176,8 +175,8 @@ badhash_cache(Config) ->
     CachedFile = filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".tar">>),
     ?assert(filelib:is_regular(CachedFile)),
     {ok, Content} = file:read_file(CachedFile),
-    ?assertMatch({unexpected_hash, _Path, ?bad_checksum, ?good_checksum},
-                 rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?bad_checksum}, State)),
+    ?assertMatch({error, {rebar_pkg_resource, {bad_registry_checksum, _, _, _, _}}},
+                 rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?bad_checksum, #{}}, State, #{}, true)),
     %% The cached file is there still, unchanged.
     ?assert(filelib:is_regular(CachedFile)),
     ?assertEqual({ok, Content}, file:read_file(CachedFile)).
@@ -190,8 +189,8 @@ bad_to_good(Config) ->
     CachedFile = filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".tar">>),
     ?assert(filelib:is_regular(CachedFile)),
     {ok, Contents} = file:read_file(CachedFile),
-    ?assertEqual({ok, true},
-                 rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum}, State)),
+    ?assertEqual(ok,
+                 rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum, #{}}, State, #{}, true)),
     %% Cache has refreshed
     ?assert({ok, Contents} =/= file:read_file(CachedFile)).
 
@@ -205,8 +204,8 @@ good_disconnect(Config) ->
     ?assert(filelib:is_regular(CachedFile)),
     {ok, Content} = file:read_file(CachedFile),
     rebar_pkg_resource:store_etag_in_cache(ETagFile, ?BADPKG_ETAG),
-    ?assertEqual({ok, true},
-                 rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum}, State)),
+    ?assertEqual(ok,
+                 rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum, #{}}, State, #{}, true)),
     {ok, Content} = file:read_file(CachedFile).
 
 bad_disconnect(Config) ->
@@ -214,33 +213,37 @@ bad_disconnect(Config) ->
     {Pkg,Vsn} = ?config(pkg, Config),
     State = ?config(state, Config),
     ?assertEqual({fetch_fail, Pkg, Vsn},
-                 rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum}, State)).
+                 rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum, #{}}, State, #{}, true)).
 
 pkgs_provider(Config) ->
     Config1 = rebar_test_utils:init_rebar_state(Config),
     rebar_test_utils:run_and_check(
-      Config1, [], ["pkgs"],
+      Config1, [], ["pkgs", "relx"],
         {ok, []}
     ).
 
 find_highest_matching(_Config) ->
     State = rebar_state:new(),
-    {ok, Vsn} = rebar_packages:find_highest_matching(
-                  <<"test">>, <<"1.0.0">>, <<"goodpkg">>, <<"1.0.0">>, package_index, State),
-    ?assertEqual(<<"1.0.1">>, Vsn),
+    {ok, Vsn} = rebar_packages:find_highest_matching_(
+                  <<"goodpkg">>, ec_semver:parse(<<"1.0.0">>), #{name => <<"hexpm">>}, ?PACKAGE_TABLE, State),
+    ?assertEqual({{1,0,1},{[],[]}}, Vsn),
     {ok, Vsn1} = rebar_packages:find_highest_matching(
-                   <<"test">>, <<"1.0.0">>, <<"goodpkg">>, <<"1.0">>, package_index, State),
-    ?assertEqual(<<"1.1.1">>, Vsn1),
+                   <<"goodpkg">>, ec_semver:parse(<<"1.0">>), #{name => <<"hexpm">>}, ?PACKAGE_TABLE, State),
+    ?assertEqual({{1,1,1},{[],[]}}, Vsn1),
     {ok, Vsn2} = rebar_packages:find_highest_matching(
-                   <<"test">>, <<"1.0.0">>, <<"goodpkg">>, <<"2.0">>, package_index, State),
-    ?assertEqual(<<"2.0.0">>, Vsn2),
+                   <<"goodpkg">>, ec_semver:parse(<<"2.0">>), #{name => <<"hexpm">>}, ?PACKAGE_TABLE, State),
+    ?assertEqual({{2,0,0},{[],[]}}, Vsn2),
 
     %% regression test. ~> constraints higher than the available packages would result
     %% in returning the first package version instead of 'none'.
-    ?assertEqual(none, rebar_packages:find_highest_matching(<<"test">>, <<"1.0.0">>, <<"goodpkg">>,
-                                                            <<"~> 5.0">>, package_index, State)).
+    ?assertEqual(none, rebar_packages:find_highest_matching_(<<"goodpkg">>, ec_semver:parse(<<"5.0">>),
+                                                             #{name => <<"hexpm">>}, ?PACKAGE_TABLE, State)),
 
 
+    {ok, Vsn3} = rebar_packages:find_highest_matching_(<<"goodpkg">>, ec_semver:parse(<<"3.0.0-rc.0">>),
+                                                       #{name => <<"hexpm">>}, ?PACKAGE_TABLE, State),
+    ?assertEqual({{3,0,0},{[<<"rc">>,0],[]}}, Vsn3).
+
 %%%%%%%%%%%%%%%
 %%% Helpers %%%
 %%%%%%%%%%%%%%%
@@ -249,35 +252,68 @@ mock_config(Name, Config) ->
     CacheRoot = filename:join([Priv, "cache", atom_to_list(Name)]),
     TmpDir = filename:join([Priv, "tmp", atom_to_list(Name)]),
     Tid = ets:new(registry_table, [public]),
-    ets:insert_new(Tid, [
-        {<<"badindexchk">>,[[<<"1.0.0">>]]},
-        {<<"goodpkg">>,[[<<"1.0.0">>, <<"1.0.1">>, <<"1.1.1">>, <<"2.0.0">>]]},
-        {<<"badpkg">>,[[<<"1.0.0">>]]},
+    AllDeps = [        
         {{<<"badindexchk">>,<<"1.0.0">>}, [[], ?bad_checksum, [<<"rebar3">>]]},
         {{<<"goodpkg">>,<<"1.0.0">>}, [[], ?good_checksum, [<<"rebar3">>]]},
         {{<<"goodpkg">>,<<"1.0.1">>}, [[], ?good_checksum, [<<"rebar3">>]]},
         {{<<"goodpkg">>,<<"1.1.1">>}, [[], ?good_checksum, [<<"rebar3">>]]},
         {{<<"goodpkg">>,<<"2.0.0">>}, [[], ?good_checksum, [<<"rebar3">>]]},
-        {{<<"badpkg">>,<<"1.0.0">>}, [[], ?good_checksum, [<<"rebar3">>]]}
-    ]),
+        {{<<"goodpkg">>,<<"3.0.0-rc.0">>}, [[], ?good_checksum, [<<"rebar3">>]]},
+        {{<<"badpkg">>,<<"1.0.0">>}, [[], ?badpkg_checksum, [<<"rebar3">>]]}
+    ],
+    ets:insert_new(Tid, AllDeps),
     CacheDir = filename:join([CacheRoot, "hex", "com", "test", "packages"]),
     filelib:ensure_dir(filename:join([CacheDir, "registry"])),
     ok = ets:tab2file(Tid, filename:join([CacheDir, "registry"])),
 
+    catch ets:delete(?PACKAGE_TABLE),
+    rebar_packages:new_package_table(),    
+    lists:foreach(fun({{N, Vsn}, [Deps, Checksum, _]}) ->
+                          case ets:member(?PACKAGE_TABLE, {ec_cnv:to_binary(N), Vsn, <<"hexpm">>}) of
+                              false ->
+                                  ets:insert(?PACKAGE_TABLE, #package{key={ec_cnv:to_binary(N), ec_semver:parse(Vsn), <<"hexpm">>},
+                                                                      dependencies=Deps,
+                                                                      retired=false,
+                                                                      checksum=Checksum});
+                              true ->
+                                  ok
+                          end
+                  end, AllDeps),
+    
+
+    meck:new(hex_repo, [passthrough]),
+    meck:expect(hex_repo, get_package, 
+                fun(_Config, PkgName) ->
+                        Matches = ets:match_object(Tid, {{PkgName,'_'}, '_'}),
+                        Releases = 
+                            [#{checksum => Checksum,
+                               version => Vsn,
+                               dependencies => Deps} || 
+                                {{_, Vsn}, [Deps, Checksum, _]} <- Matches],
+                        {ok, {200, #{}, #{releases => Releases}}}
+                end),
+
     %% The state returns us a fake registry
     meck:new(rebar_state, [passthrough]),
     meck:expect(rebar_state, get,
                 fun(_State, rebar_packages_cdn, _Default) ->
-                    "http://test.com/"
+                        "http://test.com/";
+                   (_, _, Default) ->
+                        Default
+                end),
+    meck:expect(rebar_state, resources,
+                fun(_State) ->
+                        DefaultConfig = hex_core:default_config(),
+                        [rebar_resource_v2:new(pkg, rebar_pkg_resource,
+                                               #{repos => [DefaultConfig#{name => <<"hexpm">>}],
+                                                 base_config => #{}})]
                 end),
 
     meck:new(rebar_dir, [passthrough]),
     meck:expect(rebar_dir, global_cache_dir, fun(_) -> CacheRoot end),
 
-    meck:new(rebar_packages, [passthrough]),
     meck:expect(rebar_packages, registry_dir, fun(_) -> {ok, CacheDir} end),
-    meck:expect(rebar_packages, package_dir, fun(_) -> {ok, CacheDir} end),
-    rebar_prv_update:hex_to_index(rebar_state:new()),
+    meck:expect(rebar_packages, package_dir, fun(_, _) -> {ok, CacheDir} end),
 
     meck:new(rebar_prv_update, [passthrough]),
     meck:expect(rebar_prv_update, do, fun(State) -> {ok, State} end),
@@ -285,16 +321,16 @@ mock_config(Name, Config) ->
     %% Cache fetches are mocked -- we assume the server and clients are
     %% correctly used.
     GoodCache = ?config(good_cache, Config),
-    {Pkg,Vsn} = ?config(pkg, Config),
+    {Pkg,Vsn} = ?config(pkg, Config), 
     PkgFile = <<Pkg/binary, "-", Vsn/binary, ".tar">>,
     {ok, PkgContents} = file:read_file(filename:join(?config(data_dir, Config), PkgFile)),
-    meck:new(httpc, [passthrough, unsticky]),
-    meck:expect(httpc, request,
-            fun(get, {_Url, _Opts}, _, _, _) when GoodCache ->;
-                {ok, {{Vsn, 304, <<"Not Modified">>}, [{"etag", ?good_etag}], <<>>}};
-               (get, {_Url, _Opts}, _, _, _) ->
-                {ok, {{Vsn, 200, <<"OK">>}, [{"etag", ?good_etag}], PkgContents}}
-            end),
+
+    meck:expect(hex_repo, get_tarball, fun(_, _, _) when GoodCache ->
+                                               {ok, {304, #{<<"etag">> => ?good_etag}, <<>>}};
+                                          (_, _, _) ->;
+                                               {ok, {200, #{<<"etag">> => ?good_etag}, PkgContents}}
+                                       end),   
+ 
     [{cache_root, CacheRoot},
      {cache_dir, CacheDir},
      {tmp_dir, TmpDir},

test/rebar_pkg_alias_SUITE.erl

@@ -3,44 +3,53 @@
 -include_lib("common_test/include/ct.hrl").
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("kernel/include/file.hrl").
+-include("rebar.hrl").
 
-all() -> [same_alias, diff_alias, diff_alias_vsn, transitive_alias,
-          transitive_hash_mismatch].
+all() -> [same_alias, diff_alias, diff_alias_vsn, transitive_alias%% ,
+          %% transitive_hash_mismatch
+         ].
 
 %% {uuid, {pkg, uuid}} = uuid
 %% {uuid, {pkg, alias}} = uuid on disk
 %% another run should yield the same lock file without error
 init_per_suite(Config) ->
-    mock_config(?MODULE, Config).
+    Config.
+    %% mock_config(?MODULE, Config).
 
 end_per_suite(Config) ->
-    unmock_config(Config).
+    Config.
+    %% unmock_config(Config).
 
 init_per_testcase(same_alias, Config0) ->
+    mock_config(?MODULE, Config0),
     Config = rebar_test_utils:init_rebar_state(Config0,"same_alias_"),
     AppDir = ?config(apps, Config),
     rebar_test_utils:create_app(AppDir, "A", "0.0.0", [kernel, stdlib]),
     RebarConf = rebar_test_utils:create_config(AppDir, [{deps, [{fakelib, {pkg, fakelib}}]}]),
     [{rebarconfig, RebarConf} | Config];
 init_per_testcase(diff_alias, Config0) ->
+    mock_config(?MODULE, Config0),
     Config = rebar_test_utils:init_rebar_state(Config0,"diff_alias_"),
     AppDir = ?config(apps, Config),
     rebar_test_utils:create_app(AppDir, "A", "0.0.0", [kernel, stdlib]),
     RebarConf = rebar_test_utils:create_config(AppDir, [{deps, [{fakelib, {pkg, goodpkg}}]}]),
     [{rebarconfig, RebarConf} | Config];
 init_per_testcase(diff_alias_vsn, Config0) ->
+    mock_config(?MODULE, Config0),
     Config = rebar_test_utils:init_rebar_state(Config0,"diff_alias_vsn_"),
     AppDir = ?config(apps, Config),
     rebar_test_utils:create_app(AppDir, "A", "0.0.0", [kernel, stdlib]),
     RebarConf = rebar_test_utils:create_config(AppDir, [{deps, [{fakelib, "1.0.0", {pkg, goodpkg}}]}]),
     [{rebarconfig, RebarConf} | Config];
 init_per_testcase(transitive_alias, Config0) ->
+    mock_config(?MODULE, Config0),
     Config = rebar_test_utils:init_rebar_state(Config0,"transitive_alias_vsn_"),
     AppDir = ?config(apps, Config),
     rebar_test_utils:create_app(AppDir, "A", "0.0.0", [kernel, stdlib]),
     RebarConf = rebar_test_utils:create_config(AppDir, [{deps, [{topdep, "1.0.0", {pkg, topdep}}]}]),
     [{rebarconfig, RebarConf} | Config];
 init_per_testcase(transitive_hash_mismatch, Config0) ->
+    mock_config(?MODULE, Config0),
     Config = rebar_test_utils:init_rebar_state(Config0,"transitive_alias_vsn_"),
     AppDir = ?config(apps, Config),
     rebar_test_utils:create_app(AppDir, "A", "0.0.0", [kernel, stdlib]),
@@ -48,6 +57,7 @@ init_per_testcase(transitive_hash_mismatch, Config0) ->
     [{rebarconfig, RebarConf} | Config].
 
 end_per_testcase(_, Config) ->
+    unmock_config(Config),
     Config.
 
 same_alias(Config) ->
@@ -162,6 +172,10 @@ transitive_hash_mismatch(Config) ->
     ),
     ok.
 
+parse_deps(Deps) ->
+    [{maps:get(app, D, Name), {pkg, Name, Constraint, undefined}} || D=#{package := Name,
+                                                                         requirement := Constraint} <- Deps].
+
 mock_config(Name, Config) ->
     {ChkFake, Etag} = create_lib(Name, Config, "fakelib"),
     {ChkTop, _} = create_lib(Name, Config, "topdep"),
@@ -176,40 +190,74 @@ mock_config(Name, Config) ->
     CacheDir = filename:join([CacheRoot, "hex", "com", "test", "packages"]),
     rebar_test_utils:create_app(AppDir, "fakelib", "1.0.0", [kernel, stdlib]),
     ct:pal("{~p, ~p}",[ChkFake, Etag]),
-    {ChkFake, Etag} = rebar_test_utils:package_app(AppDir, CacheDir, "goodpkg-1.0.0"),
+    {ChkGood, EtagGood} = rebar_test_utils:package_app(AppDir, CacheDir, "goodpkg", "1.0.0"),
 
+    AllDeps = [
+     {<<"fakelib">>,[[<<"1.0.0">>]]},
+     {<<"goodpkg">>,[[<<"1.0.0">>]]},
+     {<<"topdep">>,[[<<"1.0.0">>]]},
+     {<<"transitive">>, [[<<"1.0.0">>]]},
+     {{<<"fakelib">>,<<"1.0.0">>}, [[], ChkFake, [<<"rebar3">>]]},
+     {{<<"goodpkg">>,<<"1.0.0">>}, [[], ChkGood, [<<"rebar3">>]]},
+     {{<<"topdep">>,<<"1.0.0">>},
+      [[
+        {<<"transitive">>, <<"1.0.0">>, false, <<"transitive_app">>}
+       ], ChkTop, [<<"rebar3">>]]},
+     {{<<"transitive">>,<<"1.0.0">>}, [[], ChkTrans, [<<"rebar3">>]]}
+    ],
     Tid = ets:new(registry_table, [public]),
-    ets:insert_new(Tid, [
-        {<<"fakelib">>,[[<<"1.0.0">>]]},
-        {<<"goodpkg">>,[[<<"1.0.0">>]]},
-        {<<"topdep">>,[[<<"1.0.0">>]]},
-        {<<"transitive">>, [[<<"1.0.0">>]]},
-        {{<<"fakelib">>,<<"1.0.0">>}, [[], ChkFake, [<<"rebar3">>]]},
-        {{<<"goodpkg">>,<<"1.0.0">>}, [[], ChkFake, [<<"rebar3">>]]},
-        {{<<"topdep">>,<<"1.0.0">>},
-         [[
-           [<<"transitive">>, <<"1.0.0">>, false, <<"transitive_app">>]
-          ], ChkTop, [<<"rebar3">>]]},
-        {{<<"transitive">>,<<"1.0.0">>}, [[], ChkTrans, [<<"rebar3">>]]}
-    ]),
+    ets:insert_new(Tid, AllDeps),
     ok = ets:tab2file(Tid, filename:join([CacheDir, "registry"])),
-    ets:delete(Tid),
+    %% ets:delete(Tid),
     %% The state returns us a fake registry
     meck:new(rebar_dir, [passthrough, no_link]),
     meck:expect(rebar_dir, global_cache_dir, fun(_) -> CacheRoot end),
 
     meck:new(rebar_packages, [passthrough, no_link]),
     meck:expect(rebar_packages, registry_dir, fun(_) -> {ok, CacheDir} end),
-    meck:expect(rebar_packages, package_dir, fun(_) -> {ok, CacheDir} end),
-    rebar_prv_update:hex_to_index(rebar_state:new()),
-
-    %% Cache fetches are mocked -- we assume the server and clients are
-    %% correctly used.
-    meck:new(httpc, [passthrough, unsticky, no_link]),
-    meck:expect(httpc, request,
-            fun(get, {_Url, _Opts}, _, _, _) ->
-                {ok, {{<<"1.0.0">>, 304, <<"Not Modified">>}, [{"etag", Etag}], <<>>}}
-            end),
+    meck:expect(rebar_packages, package_dir, fun(_, _) -> {ok, CacheDir} end),
+
+    %% TODO: is something else wrong that we need this for transitive_alias to pass
+    meck:expect(rebar_packages, update_package, fun(_, _, _) -> ok end),
+
+    meck:new(rebar_prv_update, [passthrough]),
+    meck:expect(rebar_prv_update, do, fun(State) -> {ok, State} end),
+
+    catch ets:delete(?PACKAGE_TABLE),
+    rebar_packages:new_package_table(),    
+
+    lists:foreach(fun({{N, Vsn}, [Deps, Checksum, _]}) ->
+                          case ets:member(?PACKAGE_TABLE, {ec_cnv:to_binary(N), Vsn, <<"hexpm">>}) of
+                              false ->
+                                  ets:insert(?PACKAGE_TABLE, #package{key={ec_cnv:to_binary(N), ec_semver:parse(Vsn), <<"hexpm">>},
+                                                                      dependencies=[{DAppName, {pkg, DN, DV, undefined}} || {DN, DV, _, DAppName} <- Deps],
+                                                                      retired=false,
+                                                                      checksum=Checksum});
+                              true ->
+                                  ok
+                          end;
+                     ({_N, _Vsns}) ->
+                          ok
+                     
+                  end, AllDeps),    
+
+    meck:new(hex_repo, [passthrough]),
+    meck:expect(hex_repo, get_package, 
+                fun(_Config, PkgName) ->
+                        Matches = ets:match_object(Tid, {{PkgName,'_'}, '_'}),
+                        Releases =
+                            [#{checksum => Checksum,
+                               version => Vsn,
+                               dependencies => [{DAppName, {pkg, DN, DV, undefined}} ||
+                                                   {DN, DV, _, DAppName} <- Deps]} ||
+                                {{_, Vsn}, [Deps, Checksum, _]} <- Matches],
+                        {ok, {200, #{}, #{releases => Releases}}}
+                end),
+
+    meck:expect(hex_repo, get_tarball, fun(_, _, _) ->
+                                               {ok, {304, #{<<"etag">> => EtagGood}, <<>>}}
+                                       end),   
+    
     %% Move all packages to cache
     NewConf = [{cache_root, CacheRoot},
                {cache_dir, CacheDir},
@@ -231,4 +279,4 @@ create_lib(Name, Config, AppName, PkgName) ->
     CacheDir = filename:join([CacheRoot, "hex", "com", "test", "packages"]),
     filelib:ensure_dir(filename:join([CacheDir, "registry"])),
     rebar_test_utils:create_app(AppDir, AppName, "1.0.0", [kernel, stdlib]),
-    rebar_test_utils:package_app(AppDir, CacheDir, PkgName++"-1.0.0").
+    rebar_test_utils:package_app(AppDir, CacheDir, PkgName, "1.0.0").

test/rebar_pkg_repos_SUITE.erl

@@ -0,0 +1,376 @@
+%% Test suite for the handling hexpm repo configurations
+-module(rebar_pkg_repos_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+-include_lib("eunit/include/eunit.hrl").
+-include("rebar.hrl").
+
+all() ->
+    [default_repo, repo_merging, repo_replacing,
+     auth_merging, organization_merging, {group, resolve_version}].
+
+groups() ->
+    [{resolve_version, [use_first_repo_match, use_exact_with_hash, fail_repo_update,
+                        ignore_match_in_excluded_repo, optional_prereleases]}].
+
+init_per_group(resolve_version, Config) ->
+    Repo1 = <<"test-repo-1">>,
+    Repo2 = <<"test-repo-2">>,
+    Repo3 = <<"test-repo-3">>,
+    Hexpm = <<"hexpm">>,
+    Repos = [Repo1, Repo2, Repo3, Hexpm],
+
+    Deps = [{"A", "0.1.1", <<"good checksum">>, Repo1, false},
+            {"A", "0.1.1", <<"good checksum">>, Repo2, false},
+            {"B", "1.0.0", Repo1, false},
+            {"B", "2.0.0", Repo2, false},
+            {"B", "1.4.0", Repo3, false},
+            {"B", "1.4.3", Hexpm, false},
+            {"B", "1.4.6", Hexpm, #{reason => 'RETIRED_INVALID'}},
+            {"B", "1.5.0", Hexpm, false},
+            {"B", "1.5.6-rc.0", Hexpm, true},
+            {"C", "1.3.1", <<"bad checksum">>, Repo1, false},
+            {"C", "1.3.1", <<"good checksum">>, Repo2, false}],
+    [{deps, Deps}, {repos, Repos} | Config];
+init_per_group(_, Config) ->
+    Config.
+
+end_per_group(_, _) ->
+    ok.
+
+init_per_testcase(use_first_repo_match, Config) ->
+    Deps = ?config(deps, Config),
+    Repos = ?config(repos, Config),
+    State = setup_deps_and_repos(Deps, Repos),
+
+    meck:new(rebar_packages, [passthrough, no_link]),
+
+    %% fail when the first repo is updated since it doesn't have a matching package
+    %% should continue anyway
+    meck:expect(rebar_packages, update_package,
+                fun(_, _, _State) -> ok  end),
+    meck:expect(rebar_packages, verify_table,
+                fun(_State) -> true end),
+
+    [{state, State} | Config];
+init_per_testcase(use_exact_with_hash, Config) ->
+    Deps = ?config(deps, Config),
+    Repos = ?config(repos, Config),
+    State = setup_deps_and_repos(Deps, Repos),
+
+    meck:new(rebar_packages, [passthrough, no_link]),
+
+    %% fail when the first repo is updated since it doesn't have a matching package
+    %% should continue anyway
+    meck:expect(rebar_packages, update_package,
+                fun(_, _, _State) -> ok  end),
+    meck:expect(rebar_packages, verify_table,
+                fun(_State) -> true end),
+
+    [{state, State} | Config];
+init_per_testcase(fail_repo_update, Config) ->
+    Deps = ?config(deps, Config),
+    Repos = ?config(repos, Config),
+    State = setup_deps_and_repos(Deps, Repos),
+
+    meck:new(rebar_packages, [passthrough, no_link]),
+
+    %% fail when the first repo is updated since it doesn't have a matching package
+    %% should continue anyway
+    [Repo1 | _] = Repos,
+    meck:expect(rebar_packages, update_package,
+                fun(_, #{name := Repo}, _State) when Repo =:= Repo1 -> fail;
+                   (_, _, _State) -> ok  end),
+    meck:expect(rebar_packages, verify_table,
+                fun(_State) -> true end),
+
+    [{state, State} | Config];
+init_per_testcase(ignore_match_in_excluded_repo, Config) ->
+    Deps = ?config(deps, Config),
+    Repos = [Repo1, _, Repo3 | _] = ?config(repos, Config),
+
+    %% drop repo1 and repo2 from the repos to be used by the pkg resource
+    State = setup_deps_and_repos(Deps, [R || R <- Repos, R =/= Repo3, R =/= Repo1]),
+
+    meck:new(rebar_packages, [passthrough, no_link]),
+
+    %% fail when the first repo is updated since it doesn't have a matching package
+    %% should continue anyway
+    [_, _, Repo3 | _] = Repos,
+    meck:expect(rebar_packages, update_package,
+                fun(_, _, _State) -> ok  end),
+    meck:expect(rebar_packages, verify_table,
+                fun(_State) -> true end),
+
+    [{state, State} | Config];
+init_per_testcase(optional_prereleases, Config) ->
+    Deps = ?config(deps, Config),
+    Repos = ?config(repos, Config),
+
+    State = setup_deps_and_repos(Deps, Repos),
+
+    meck:new(rebar_packages, [passthrough, no_link]),
+
+    meck:expect(rebar_packages, update_package,
+                fun(_, _, _State) -> ok  end),
+    meck:expect(rebar_packages, verify_table,
+                fun(_State) -> true end),
+
+    [{state, State} | Config];
+init_per_testcase(auth_merging, Config) ->
+    meck:new(file, [passthrough, no_link, unstick]),
+    meck:new(rebar_packages, [passthrough, no_link]),
+    Config;
+init_per_testcase(organization_merging, Config) ->
+    meck:new(file, [passthrough, no_link, unstick]),
+    meck:new(rebar_packages, [passthrough, no_link]),
+    Config;
+init_per_testcase(_, Config) ->
+    Config.
+
+end_per_testcase(Case, _Config) when Case =:= auth_merging ;
+                                     Case =:= organization_merging ->
+    meck:unload(file),
+    meck:unload(rebar_packages);
+end_per_testcase(Case, _Config) when Case =:= use_first_repo_match ;
+                                     Case =:= use_exact_with_hash ;
+                                     Case =:= fail_repo_update ;
+                                     Case =:= ignore_match_in_excluded_repo ;
+                                     Case =:= optional_prereleases ->
+    meck:unload(rebar_packages);
+end_per_testcase(_, _) ->
+    ok.
+
+
+default_repo(_Config) ->
+    Repo1 = #{name => <<"hexpm">>,
+              api_key => <<"asdf">>},
+
+    MergedRepos = rebar_hex_repos:repos([{repos, [Repo1]}]),
+
+    ?assertMatch([#{name := <<"hexpm">>,
+                    api_key := <<"asdf">>,
+                    api_url := <<"https://hex.pm/api">>}], MergedRepos).
+
+
+repo_merging(_Config) ->
+    Repo1 = #{name => <<"repo-1">>,
+              api_url => <<"repo-1/api">>},
+    Repo2 = #{name => <<"repo-2">>,
+              repo_url => <<"repo-2/repo">>,
+              repo_verify => false},
+    Result = rebar_hex_repos:merge_repos([Repo1, Repo2,
+                                             #{name => <<"repo-2">>,
+                                               api_url => <<"repo-2/api">>,
+                                               repo_url => <<"bad url">>,
+                                               repo_verify => true},
+                                             #{name => <<"repo-1">>,
+                                               api_url => <<"bad url">>,
+                                               repo_verify => true},
+                                             #{name => <<"repo-2">>,
+                                               api_url => <<"repo-2/api-2">>,
+                                               repo_url => <<"other/repo">>}]),
+    ?assertMatch([#{name := <<"repo-1">>,
+                    api_url := <<"repo-1/api">>,
+                    repo_verify := true},
+                  #{name := <<"repo-2">>,
+                    api_url := <<"repo-2/api">>,
+                    repo_url := <<"repo-2/repo">>,
+                    repo_verify := false}], Result).
+
+repo_replacing(_Config) ->
+    Repo1 = #{name => <<"repo-1">>,
+              api_url => <<"repo-1/api">>},
+    Repo2 = #{name => <<"repo-2">>,
+              repo_url => <<"repo-2/repo">>,
+              repo_verify => false},
+
+    ?assertMatch([Repo1, Repo2, #{name := <<"hexpm">>}],
+                 rebar_hex_repos:repos([{repos, [Repo1]},
+                                           {repos, [Repo2]}])),
+
+    %% use of replace is ignored if found in later entries than the first
+    ?assertMatch([Repo1, Repo2, #{name := <<"hexpm">>}],
+                 rebar_hex_repos:repos([{repos, [Repo1]},
+                                           {repos, replace, [Repo2]}])),
+
+    ?assertMatch([Repo1],
+                 rebar_hex_repos:repos([{repos, replace, [Repo1]},
+                                           {repos, [Repo2]}])).
+
+auth_merging(_Config) ->
+    Repo1 = #{name => <<"repo-1">>,
+              api_url => <<"repo-1/api">>},
+    Repo2 = #{name => <<"repo-2">>,
+              repo_url => <<"repo-2/repo">>,
+              repo_verify => false},
+
+    State = rebar_state:new([{hex, [{repos, [Repo1, Repo2]}]}]),
+    meck:expect(file, consult,
+                fun(_) ->
+                        {ok, [#{<<"repo-1">> => #{read_key => <<"read key">>,
+                                                 write_key => <<"write key">>},
+                               <<"repo-2">> => #{read_key => <<"read key 2">>,
+                                                 repos_key => <<"repos key 2">>,
+                                                 write_key => <<"write key 2">>},
+                                <<"hexpm">> => #{write_key => <<"write key hexpm">>}}]}
+                end),
+
+    ?assertMatch({ok,
+                  #resource{state=#{repos := [#{name := <<"repo-1">>,
+                                                read_key := <<"read key">>,
+                                                write_key := <<"write key">>},
+                                              #{name := <<"repo-2">>,
+                                                read_key := <<"read key 2">>,
+                                                repos_key := <<"repos key 2">>,
+                                                write_key := <<"write key 2">>},
+                                              #{name := <<"hexpm">>,
+                                                write_key := <<"write key hexpm">>}]}}},
+                 rebar_pkg_resource:init(pkg, State)),
+
+    ok.
+
+organization_merging(_Config) ->
+    Repo1 = #{name => <<"hexpm:repo-1">>,
+              api_url => <<"repo-1/api">>},
+    Repo2 = #{name => <<"hexpm:repo-2">>,
+              repo_url => <<"repo-2/repo">>,
+              repo_verify => false},
+
+    State = rebar_state:new([{hex, [{repos, [Repo1, Repo2]}]}]),
+    meck:expect(file, consult,
+                fun(_) ->
+                        {ok, [#{<<"hexpm:repo-1">> => #{read_key => <<"read key">>},
+                                <<"hexpm:repo-2">> => #{read_key => <<"read key 2">>,
+                                                        repos_key => <<"repos key 2">>,
+                                                        write_key => <<"write key 2">>},
+                                <<"hexpm">> => #{write_key => <<"write key hexpm">>}}]}
+                end),
+
+    ?assertMatch({ok,
+                  #resource{state=#{repos := [#{name := <<"hexpm:repo-1">>,
+                                    parent := <<"hexpm">>,
+                                    read_key := <<"read key">>,
+                                    write_key := <<"write key hexpm">>},
+                                  #{name := <<"hexpm:repo-2">>,
+                                    parent := <<"hexpm">>,
+                                    read_key := <<"read key 2">>,
+                                    repos_key := <<"repos key 2">>,
+                                    write_key := <<"write key 2">>},
+                                  #{name := <<"hexpm">>,
+                                    write_key := <<"write key hexpm">>}]}}},
+                 rebar_pkg_resource:init(pkg, State)),
+
+    ok.
+
+use_first_repo_match(Config) ->
+    State = ?config(state, Config),
+
+    ?assertMatch({ok,{package,{<<"B">>, {{2,0,0}, {[],[]}}, Repo2},
+                      <<"some checksum">>, false, []},
+                  #{name := Repo2,
+                    http_adapter_config := #{profile := rebar}}},
+                 rebar_packages:resolve_version(<<"B">>, <<"> 1.4.0">>, undefined,
+                                                ?PACKAGE_TABLE, State)),
+
+    ?assertMatch({ok,{package,{<<"B">>, {{1,4,0}, {[],[]}}, Repo3},
+                      <<"some checksum">>, false, []},
+                  #{name := Repo3,
+                    http_adapter_config := #{profile := rebar}}},
+                 rebar_packages:resolve_version(<<"B">>, <<"~> 1.4.0">>, undefined,
+                                                ?PACKAGE_TABLE, State)).
+
+%% tests that even though an easier repo has C-1.3.1 it doesn't use it since its hash is different
+use_exact_with_hash(Config) ->
+    State = ?config(state, Config),
+
+    ?assertMatch({ok,{package,{<<"C">>, {{1,3,1}, {[],[]}}, Repo2},
+                      <<"good checksum">>, false, []},
+                  #{name := Repo2,
+                    http_adapter_config := #{profile := rebar}}},
+                 rebar_packages:resolve_version(<<"C">>, <<"1.3.1">>, <<"good checksum">>,
+                                                ?PACKAGE_TABLE, State)).
+
+fail_repo_update(Config) ->
+    State = ?config(state, Config),
+
+    ?assertMatch({ok,{package,{<<"B">>, {{1,4,0}, {[],[]}}, Repo3},
+                      <<"some checksum">>, false, []},
+                  #{name := Repo3,
+                    http_adapter_config := #{profile := rebar}}},
+                 rebar_packages:resolve_version(<<"B">>, <<"~> 1.4.0">>, undefined,
+                                                ?PACKAGE_TABLE, State)).
+
+ignore_match_in_excluded_repo(Config) ->
+    State = ?config(state, Config),
+    Repos = ?config(repos, Config),
+
+    ?assertMatch({ok,{package,{<<"B">>, {{1,4,6}, {[],[]}}, Hexpm},
+                      <<"some checksum">>, #{reason := 'RETIRED_INVALID'}, []},
+                  #{name := Hexpm,
+                    http_adapter_config := #{profile := rebar}}},
+                 rebar_packages:resolve_version(<<"B">>, <<"~> 1.4.0">>, undefined,
+                                                ?PACKAGE_TABLE, State)),
+
+    [_, Repo2 | _] = Repos,
+    ?assertMatch({ok,{package,{<<"A">>, {{0,1,1}, {[],[]}}, Repo2},
+                      <<"good checksum">>, false, []},
+                  #{name := Repo2,
+                    http_adapter_config := #{profile := rebar}}},
+                 rebar_packages:resolve_version(<<"A">>, <<"0.1.1">>, <<"good checksum">>,
+                                                ?PACKAGE_TABLE, State)).
+
+optional_prereleases(Config) ->
+    State = ?config(state, Config),
+
+    ?assertMatch({ok,{package,{<<"B">>, {{1,5,0}, {[],[]}}, Hexpm},
+                      <<"some checksum">>, false, []},
+                  #{name := Hexpm,
+                    http_adapter_config := #{profile := rebar}}},
+                 rebar_packages:resolve_version(<<"B">>, <<"~> 1.5.0">>, undefined,
+                                                ?PACKAGE_TABLE, State)),
+
+    ?assertMatch({ok,{package,{<<"B">>, {{1,5,6}, {[<<"rc">>,0],[]}}, Hexpm},
+                      <<"some checksum">>, true, []},
+                  #{name := Hexpm,
+                    http_adapter_config := #{profile := rebar}}},
+                 rebar_packages:resolve_version(<<"B">>, <<"1.5.6-rc.0">>, <<"some checksum">>,
+                                                ?PACKAGE_TABLE, State)),
+
+    %% allow prerelease through configuration
+    State1 = rebar_state:set(State, deps_allow_prerelease, true),
+    ?assertMatch({ok,{package,{<<"B">>, {{1,5,6}, {[<<"rc">>,0],[]}}, Hexpm},
+                      <<"some checksum">>, true, []},
+                  #{name := Hexpm,
+                    http_adapter_config := #{profile := rebar}}},
+                 rebar_packages:resolve_version(<<"B">>, <<"~> 1.5.0">>, <<"some checksum">>,
+                                                ?PACKAGE_TABLE, State1)).
+
+%%
+
+setup_deps_and_repos(Deps, Repos) ->
+    catch ets:delete(?PACKAGE_TABLE),
+    true = rebar_packages:new_package_table(),
+    insert_deps(Deps),
+    State = rebar_state:new([{hex, [{repos, [#{name => R} || R <- Repos]}]}]),
+    rebar_state:create_resources([{pkg, rebar_pkg_resource}], State).
+
+
+insert_deps(Deps) ->
+    lists:foreach(fun({Name, Version, Repo, Retired}) ->
+                          ets:insert(?PACKAGE_TABLE, #package{key={rebar_utils:to_binary(Name),
+                                                                   ec_semver:parse(Version),
+                                                                   rebar_utils:to_binary(Repo)},
+                                                              dependencies=[],
+                                                              retired=Retired,
+                                                              checksum = <<"some checksum">>});
+                     ({Name, Version, Checksum, Repo, Retired}) ->
+                          ets:insert(?PACKAGE_TABLE, #package{key={rebar_utils:to_binary(Name),
+                                                                   ec_semver:parse(Version),
+                                                                   rebar_utils:to_binary(Repo)},
+                                                              dependencies=[],
+                                                              retired=Retired,
+                                                              checksum = Checksum})
+                  end, Deps).

test/rebar_plugins_SUITE.erl

@@ -335,7 +335,7 @@ project_plugins(Config) ->
     rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]),
 
     DepName = rebar_test_utils:create_random_name("dep1_"),
-    PluginName = "compile",
+    PluginName = "compile_plugin",
     PluginName2 = "release",
 
     Plugins = rebar_test_utils:expand_deps(git, [{PluginName, Vsn, []}, {PluginName2, Vsn, []}]),

test/rebar_resource_SUITE.erl

@@ -29,12 +29,15 @@ init_per_testcase(change_type_upgrade, Config) ->
     TypeStr = atom_to_list(Type),
     DirName = filename:join([?config(priv_dir, Config), "resource_"++TypeStr]),
     ec_file:mkdir_path(DirName),
-    [{path, DirName} | Config].
+
+    {ok, AppInfo} = rebar_app_info:new(test_app, "0.0.1", DirName),
+    AppInfo1 = rebar_app_info:source(AppInfo, ?config(resource, Config)),
+
+    [{app, AppInfo1} | Config].
 
 end_per_testcase(_, Config) ->
     Config.
 
 change_type_upgrade(Config) ->
-    ?assert(rebar_fetch:needs_update(?config(path, Config),
-                                     ?config(resource, Config),
+    ?assert(rebar_fetch:needs_update(?config(app, Config),
                                      ?config(state, Config))).

test/rebar_test_utils.erl

@@ -1,11 +1,12 @@
 -module(rebar_test_utils).
 -include_lib("common_test/include/ct.hrl").
 -include_lib("eunit/include/eunit.hrl").
--export([init_rebar_state/1, init_rebar_state/2, run_and_check/4, check_results/3]).
+-export([init_rebar_state/1, init_rebar_state/2, run_and_check/3, run_and_check/4, check_results/3]).
 -export([expand_deps/2, flat_deps/1, top_level_deps/1]).
 -export([create_app/4, create_plugin/4, create_eunit_app/4, create_empty_app/4,
-         create_config/2, create_config/3, package_app/3]).
--export([create_random_name/1, create_random_vsn/0, write_src_file/2]).
+         create_config/2, create_config/3, package_app/4]).
+-export([create_random_name/1, create_random_vsn/0, write_src_file/2,
+         random_element/1]).
 
 %% Pick the right random module
 -ifdef(rand_only).
@@ -34,8 +35,10 @@ init_rebar_state(Config, Name) ->
     Verbosity = rebar3:log_level(),
     rebar_log:init(command_line, Verbosity),
     GlobalDir = filename:join([DataDir, "cache"]),
+    Repos = proplists:get_value(repos, Config, []),
     State = rebar_state:new([{base_dir, filename:join([AppsDir, "_build"])}
                             ,{global_rebar_dir, GlobalDir}
+                            ,{hex, [{repos, [#{name => R} || R <- Repos]}]}
                             ,{root_dir, AppsDir}]),
     [{apps, AppsDir}, {checkouts, CheckoutsDir}, {state, State} | Config].
 
@@ -79,6 +82,33 @@ run_and_check(Config, RebarConfig, Command, Expect) ->
         rebar_abort when Expect =:= rebar_abort -> rebar_abort
     end.
 
+run_and_check(Config, Command, Expect) ->
+    %% Assumes init_rebar_state has run first
+    AppDir = ?config(apps, Config),
+    {ok, Cwd} = file:get_cwd(),
+    try
+        ok = file:set_cwd(AppDir),
+        Res = rebar3:run(Command),
+        case Expect of
+            {error, Reason} ->
+                ?assertEqual({error, Reason}, Res);
+            {ok, Expected} ->
+                {ok, _} = Res,
+                check_results(AppDir, Expected, "*"),
+                Res;
+            {ok, Expected, ProfileRun} ->
+                {ok, _} = Res,
+                check_results(AppDir, Expected, ProfileRun),
+                Res;
+            return ->
+                Res
+        end
+    catch
+        rebar_abort when Expect =:= rebar_abort -> rebar_abort
+    after
+        ok = file:set_cwd(Cwd)
+    end.
+
 %% @doc Creates a dummy application including:
 %% - src/<file>.erl
 %% - src/<file>.app.src
@@ -263,6 +293,14 @@ check_results(AppDir, Expected, ProfileRun) ->
                         ok
                 end
         ; ({dep_not_exist, Name}) ->
+                ct:pal("Dep Not Exist Name: ~p", [Name]),
+                case lists:keyfind(Name, 1, DepsNames) of
+                    false ->
+                        ok;
+                    {Name, _App} ->
+                        error({app_found, Name})
+                end
+        ; ({app_not_exist, Name}) ->
                 ct:pal("App Not Exist Name: ~p", [Name]),
                 case lists:keyfind(Name, 1, DepsNames) of
                     false ->
@@ -467,24 +505,25 @@ get_app_metadata(Name, Vsn, Deps) ->
       {registered, []},
       {applications, Deps}]}.
 
-package_app(AppDir, DestDir, PkgName) ->
-    Name = PkgName++".tar",
-    {ok, Fs} = rebar_utils:list_dir(AppDir),
-    ok = erl_tar:create(filename:join(DestDir, "contents.tar.gz"),
-                        lists:zip(Fs, [filename:join(AppDir,F) || F <- Fs]),
-                        [compressed]),
-    ok = file:write_file(filename:join(DestDir, "metadata.config"), "who cares"),
-    ok = file:write_file(filename:join(DestDir, "VERSION"), "3"),
-    {ok, Contents} = file:read_file(filename:join(DestDir, "contents.tar.gz")),
-    Blob = <<"3who cares", Contents/binary>>,
-    <<X:256/big-unsigned>> = crypto:hash(sha256, Blob),
-    BinChecksum = list_to_binary(rebar_string:uppercase(lists:flatten(io_lib:format("~64.16.0b", [X])))),
-    ok = file:write_file(filename:join(DestDir, "CHECKSUM"), BinChecksum),
-    PkgFiles = ["contents.tar.gz", "VERSION", "metadata.config", "CHECKSUM"],
+package_app(AppDir, DestDir, PkgName, PkgVsn) ->
+    AppSrc = filename:join(AppDir, "src"),
+    {ok, Fs} = rebar_utils:list_dir(AppSrc),
+    Files = lists:zip([filename:join("src", F) || F <- Fs], [filename:join(AppSrc,F) || F <- Fs]),
+    Metadata = #{<<"app">> => list_to_binary(PkgName),
+                 <<"version">> => list_to_binary(PkgVsn)},
+    {ok, {Tarball, <<Checksum:256/big-unsigned-integer>>}} = hex_tarball:create(Metadata, Files),
+
+    Name = PkgName++"-"++PkgVsn++".tar",
     Archive = filename:join(DestDir, Name),
-    ok = erl_tar:create(Archive,
-                        lists:zip(PkgFiles, [filename:join(DestDir,F) || F <- PkgFiles])),
-    {ok, BinFull} = file:read_file(Archive),
-    <<E:128/big-unsigned-integer>> = crypto:hash(md5, BinFull),
-    Etag = rebar_string:lowercase(lists:flatten(io_lib:format("~32.16.0b", [E]))),
-    {BinChecksum, Etag}.
+    file:write_file(Archive, Tarball),
+
+    <<E:128/big-unsigned-integer>> = crypto:hash(md5, Tarball),
+
+    Checksum1 = list_to_binary(
+                  rebar_string:uppercase(
+                    lists:flatten(io_lib:format("~64.16.0b", [Checksum])))),
+    {Checksum1, E}.
+
+random_element(Repos) ->
+    Index = ?random:uniform(length(Repos)),
+    lists:nth(Index, Repos).

test/rebar_upgrade_SUITE.erl

@@ -112,25 +112,25 @@ setup_project(Case, Config0, Deps, UpDeps) ->
 
 upgrades(top_a) ->
      %% Original tree
-    {[{"A", "1", [{"B", [{"D", "1", []}]},
-                  {"C", [{"D", "2", []}]}]}
+    {[{"A", "1.0.0", [{"B", [{"D", "1.0.0", []}]},
+                  {"C", [{"D", "2.0.0", []}]}]}
      ],
      %% Updated tree
-     [{"A", "1", [{"B", [{"D", "3", []}]},
-                  {"C", [{"D", "2", []}]}]}
+     [{"A", "1.0.0", [{"B", [{"D", "3.0.0", []}]},
+                  {"C", [{"D", "2.0.0", []}]}]}
      ],
      %% Modified apps, gobally
      ["A","B","D"],
      %% upgrade vs. new tree
-     {"A", [{"A","1"}, "B", "C", {"D","3"}]}};
+     {"A", [{"A","1.0.0"}, "B", "C", {"D","3.0.0"}]}};
 upgrades(top_b) ->
      %% Original tree
-    {[{"A", "1", [{"B", [{"D", "1", []}]},
-                  {"C", [{"D", "2", []}]}]}
+    {[{"A", "1.0.0", [{"B", [{"D", "1.0.0", []}]},
+                  {"C", [{"D", "2.0.0", []}]}]}
      ],
      %% Updated tree
-     [{"A", "1", [{"B", [{"D", "3", []}]},
-                  {"C", [{"D", "2", []}]}]}
+     [{"A", "1.0.0", [{"B", [{"D", "3.0.0", []}]},
+                  {"C", [{"D", "2.0.0", []}]}]}
      ],
      %% Modified apps, gobally
      ["A","B","D"],
@@ -138,12 +138,12 @@ upgrades(top_b) ->
      {"B", {error, {rebar_prv_upgrade, {transitive_dependency, <<"B">>}}}}};
 upgrades(top_c) ->
      %% Original tree
-    {[{"A", "1", [{"B", [{"D", "1", []}]},
-                  {"C", [{"D", "2", []}]}]}
+    {[{"A", "1.0.0", [{"B", [{"D", "1.0.0", []}]},
+                  {"C", [{"D", "2.0.0", []}]}]}
      ],
      %% Updated tree
-     [{"A", "1", [{"B", [{"D", "3", []}]},
-                  {"C", [{"D", "2", []}]}]}
+     [{"A", "1.0.0", [{"B", [{"D", "3.0.0", []}]},
+                  {"C", [{"D", "2.0.0", []}]}]}
      ],
      %% Modified apps, gobally
      ["A","B","D"],
@@ -151,12 +151,12 @@ upgrades(top_c) ->
      {"C", {error, {rebar_prv_upgrade, {transitive_dependency, <<"C">>}}}}};
 upgrades(top_d1) ->
      %% Original tree
-    {[{"A", "1", [{"B", [{"D", "1", []}]},
-                  {"C", [{"D", "2", []}]}]}
+    {[{"A", "1.0.0", [{"B", [{"D", "1.0.0", []}]},
+                  {"C", [{"D", "2.0.0", []}]}]}
      ],
      %% Updated tree
-     [{"A", "1", [{"B", [{"D", "3", []}]},
-                  {"C", [{"D", "2", []}]}]}
+     [{"A", "1.0.0", [{"B", [{"D", "3.0.0", []}]},
+                  {"C", [{"D", "2.0.0", []}]}]}
      ],
      %% Modified apps, gobally
      ["A","B","D"],
@@ -164,12 +164,12 @@ upgrades(top_d1) ->
      {"D", {error, {rebar_prv_upgrade, {transitive_dependency, <<"D">>}}}}};
 upgrades(top_d2) ->
      %% Original tree
-    {[{"A", "1", [{"B", [{"D", "1", []}]},
-                  {"C", [{"D", "2", []}]}]}
+    {[{"A", "1.0.0", [{"B", [{"D", "1.0.0", []}]},
+                  {"C", [{"D", "2.0.0", []}]}]}
      ],
      %% Updated tree
-     [{"A", "1", [{"B", [{"D", "3", []}]},
-                  {"C", [{"D", "2", []}]}]}
+     [{"A", "1.0.0", [{"B", [{"D", "3.0.0", []}]},
+                  {"C", [{"D", "2.0.0", []}]}]}
      ],
      %% Modified apps, gobally
      ["A","B","D"],
@@ -177,342 +177,342 @@ upgrades(top_d2) ->
      {"D", {error, {rebar_prv_upgrade, {transitive_dependency, <<"D">>}}}}};
 upgrades(top_e) ->
      %% Original tree
-    {[{"A", "1", [{"B", [{"D", "1", []}]},
-                  {"C", [{"D", "2", []}]}]}
+    {[{"A", "1.0.0", [{"B", [{"D", "1.0.0", []}]},
+                  {"C", [{"D", "2.0.0", []}]}]}
      ],
      %% Updated tree
-     [{"A", "1", [{"B", [{"D", "3", []}]},
-                  {"C", [{"D", "2", []}]}]}
+     [{"A", "1.0.0", [{"B", [{"D", "3.0.0", []}]},
+                  {"C", [{"D", "2.0.0", []}]}]}
      ],
      %% Modified apps, gobally
      ["A","B","D"],
      %% upgrade vs. new tree
      {"E", {error, {rebar_prv_upgrade, {unknown_dependency, <<"E">>}}}}};
 upgrades(pair_a) ->
-    {[{"A", "1", [{"C", "1", []}]},
-      {"B", "1", [{"D", "1", []}]}
+    {[{"A", "1.0.0", [{"C", "1.0.0", []}]},
+      {"B", "1.0.0", [{"D", "1.0.0", []}]}
      ],
-     [{"A", "2", [{"C", "2", []}]},
-      {"B", "2", [{"D", "2", []}]}
+     [{"A", "2.0.0", [{"C", "2.0.0", []}]},
+      {"B", "2.0.0", [{"D", "2.0.0", []}]}
      ],
      ["A","B","C","D"],
-     {"A", [{"A","2"},{"C","2"},{"B","1"},{"D","1"}]}};
+     {"A", [{"A","2.0.0"},{"C","2.0.0"},{"B","1.0.0"},{"D","1.0.0"}]}};
 upgrades(pair_b) ->
-    {[{"A", "1", [{"C", "1", []}]},
-      {"B", "1", [{"D", "1", []}]}
+    {[{"A", "1.0.0", [{"C", "1.0.0", []}]},
+      {"B", "1.0.0", [{"D", "1.0.0", []}]}
      ],
-     [{"A", "2", [{"C", "2", []}]},
-      {"B", "2", [{"D", "2", []}]}
+     [{"A", "2.0.0", [{"C", "2.0.0", []}]},
+      {"B", "2.0.0", [{"D", "2.0.0", []}]}
      ],
      ["A","B","C","D"],
-     {"B", [{"A","1"},{"C","1"},{"B","2"},{"D","2"}]}};
+     {"B", [{"A","1.0.0"},{"C","1.0.0"},{"B","2.0.0"},{"D","2.0.0"}]}};
 upgrades(pair_ab) ->
-    {[{"A", "1", [{"C", "1", []}]},
-      {"B", "1", [{"D", "1", []}]}
+    {[{"A", "1.0.0", [{"C", "1.0.0", []}]},
+      {"B", "1.0.0", [{"D", "1.0.0", []}]}
      ],
-     [{"A", "2", [{"C", "2", []}]},
-      {"B", "2", [{"D", "2", []}]}
+     [{"A", "2.0.0", [{"C", "2.0.0", []}]},
+      {"B", "2.0.0", [{"D", "2.0.0", []}]}
      ],
      ["A","B","C","D"],
-     {"A,B", [{"A","2"},{"C","2"},{"B","2"},{"D","2"}]}};
+     {"A,B", [{"A","2.0.0"},{"C","2.0.0"},{"B","2.0.0"},{"D","2.0.0"}]}};
 upgrades(pair_c) ->
-    {[{"A", "1", [{"C", "1", []}]},
-      {"B", "1", [{"D", "1", []}]}
+    {[{"A", "1.0.0", [{"C", "1.0.0", []}]},
+      {"B", "1.0.0", [{"D", "1.0.0", []}]}
      ],
-     [{"A", "2", [{"C", "2", []}]},
-      {"B", "2", [{"D", "2", []}]}
+     [{"A", "2.0.0", [{"C", "2.0.0", []}]},
+      {"B", "2.0.0", [{"D", "2.0.0", []}]}
      ],
      ["A","B","C","D"],
      {"C", {error, {rebar_prv_upgrade, {transitive_dependency, <<"C">>}}}}};
 upgrades(pair_all) ->
-    {[{"A", "1", [{"C", "1", []}]},
-      {"B", "1", [{"D", "1", []}]}
+    {[{"A", "1.0.0", [{"C", "1.0.0", []}]},
+      {"B", "1.0.0", [{"D", "1.0.0", []}]}
      ],
-     [{"A", "2", [{"C", "2", []}]},
-      {"B", "2", [{"D", "2", []}]}
+     [{"A", "2.0.0", [{"C", "2.0.0", []}]},
+      {"B", "2.0.0", [{"D", "2.0.0", []}]}
      ],
      ["A","B","C","D"],
-     {"", [{"A","2"},{"C","2"},{"B","2"},{"D","2"}]}};
+     {"", [{"A","2.0.0"},{"C","2.0.0"},{"B","2.0.0"},{"D","2.0.0"}]}};
 upgrades(triplet_a) ->
-    {[{"A", "1", [{"D",[]},
-                  {"E","3",[]}]},
-      {"B", "1", [{"F","1",[]},
+    {[{"A", "1.0.0", [{"D",[]},
+                  {"E","3.0.0",[]}]},
+      {"B", "1.0.0", [{"F","1.0.0",[]},
                   {"G",[]}]},
-      {"C", "0", [{"H","3",[]},
+      {"C", "0.0.0", [{"H","3.0.0",[]},
                   {"I",[]}]}],
-     [{"A", "1", [{"D",[]},
-                  {"E","2",[]}]},
-      {"B", "1", [{"F","1",[]},
+     [{"A", "1.0.0", [{"D",[]},
+                  {"E","2.0.0",[]}]},
+      {"B", "1.0.0", [{"F","1.0.0",[]},
                   {"G",[]}]},
-      {"C", "1", [{"H","4",[]},
+      {"C", "1.0.0", [{"H","4.0.0",[]},
                   {"I",[]}]}],
      ["A","C","E","H"],
-     {"A", [{"A","1"}, "D", {"E","2"},
-            {"B","1"}, {"F","1"}, "G",
-            {"C","0"}, {"H","3"}, "I"]}};
+     {"A", [{"A","1.0.0"}, "D", {"E","2.0.0"},
+            {"B","1.0.0"}, {"F","1.0.0"}, "G",
+            {"C","0.0.0"}, {"H","3.0.0"}, "I"]}};
 upgrades(triplet_b) ->
-    {[{"A", "1", [{"D",[]},
-                  {"E","3",[]}]},
-      {"B", "1", [{"F","1",[]},
+    {[{"A", "1.0.0", [{"D",[]},
+                  {"E","3.0.0",[]}]},
+      {"B", "1.0.0", [{"F","1.0.0",[]},
                   {"G",[]}]},
-      {"C", "0", [{"H","3",[]},
+      {"C", "0.0.0", [{"H","3.0.0",[]},
                   {"I",[]}]}],
-     [{"A", "2", [{"D",[]},
-                  {"E","2",[]}]},
-      {"B", "1", [{"F","1",[]},
+     [{"A", "2.0.0", [{"D",[]},
+                  {"E","2.0.0",[]}]},
+      {"B", "1.0.0", [{"F","1.0.0",[]},
                   {"G",[]}]},
-      {"C", "1", [{"H","4",[]},
+      {"C", "1.0.0", [{"H","4.0.0",[]},
                   {"I",[]}]}],
      ["A","C","E","H"],
-     {"B", [{"A","1"}, "D", {"E","3"},
-            {"B","1"}, {"F","1"}, "G",
-            {"C","0"}, {"H","3"}, "I"]}};
+     {"B", [{"A","1.0.0"}, "D", {"E","3.0.0"},
+            {"B","1.0.0"}, {"F","1.0.0"}, "G",
+            {"C","0.0.0"}, {"H","3.0.0"}, "I"]}};
 upgrades(triplet_c) ->
-    {[{"A", "1", [{"D",[]},
-                  {"E","3",[]}]},
-      {"B", "1", [{"F","1",[]},
+    {[{"A", "1.0.0", [{"D",[]},
+                  {"E","3.0.0",[]}]},
+      {"B", "1.0.0", [{"F","1.0.0",[]},
                   {"G",[]}]},
-      {"C", "0", [{"H","3",[]},
+      {"C", "0.0.0", [{"H","3.0.0",[]},
                   {"I",[]}]}],
-     [{"A", "2", [{"D",[]},
-                  {"E","2",[]}]},
-      {"B", "1", [{"F","1",[]},
+     [{"A", "2.0.0", [{"D",[]},
+                  {"E","2.0.0",[]}]},
+      {"B", "1.0.0", [{"F","1.0.0",[]},
                   {"G",[]}]},
-      {"C", "1", [{"H","4",[]},
+      {"C", "1.0.0", [{"H","4.0.0",[]},
                   {"I",[]}]}],
      ["A","C","E","H"],
-     {"C", [{"A","1"}, "D", {"E","3"},
-            {"B","1"}, {"F","1"}, "G",
-            {"C","1"}, {"H","4"}, "I"]}};
+     {"C", [{"A","1.0.0"}, "D", {"E","3.0.0"},
+            {"B","1.0.0"}, {"F","1.0.0"}, "G",
+            {"C","1.0.0"}, {"H","4.0.0"}, "I"]}};
 upgrades(tree_a) ->
-    {[{"A", "1", [{"D",[{"J",[]}]},
-                  {"E",[{"I","1",[]}]}]},
-      {"B", "1", [{"F",[]},
+    {[{"A", "1.0.0", [{"D",[{"J",[]}]},
+                  {"E",[{"I","1.0.0",[]}]}]},
+      {"B", "1.0.0", [{"F",[]},
                   {"G",[]}]},
-      {"C", "1", [{"H",[]},
-                  {"I","2",[]}]}
+      {"C", "1.0.0", [{"H",[]},
+                  {"I","2.0.0",[]}]}
      ],
-     [{"A", "1", [{"D",[{"J",[]}]},
-                  {"E",[{"I","1",[]}]}]},
-      {"B", "1", [{"F",[]},
+     [{"A", "1.0.0", [{"D",[{"J",[]}]},
+                  {"E",[{"I","1.0.0",[]}]}]},
+      {"B", "1.0.0", [{"F",[]},
                   {"G",[]}]},
-      {"C", "2", [{"H",[]}]}
+      {"C", "2.0.0", [{"H",[]}]}
      ],
      ["C"],
-     {"A", [{"A","1"}, "D", "J", "E",
-            {"B","1"}, "F", "G",
-            {"C","1"}, "H", {"I","2"}]}};
+     {"A", [{"A","1.0.0"}, "D", "J", "E",
+            {"B","1.0.0"}, "F", "G",
+            {"C","1.0.0"}, "H", {"I","2.0.0"}]}};
 upgrades(tree_b) ->
-    {[{"A", "1", [{"D",[{"J",[]}]},
-                  {"E",[{"I","1",[]}]}]},
-      {"B", "1", [{"F",[]},
+    {[{"A", "1.0.0", [{"D",[{"J",[]}]},
+                  {"E",[{"I","1.0.0",[]}]}]},
+      {"B", "1.0.0", [{"F",[]},
                   {"G",[]}]},
-      {"C", "1", [{"H",[]},
-                  {"I","2",[]}]}
+      {"C", "1.0.0", [{"H",[]},
+                  {"I","2.0.0",[]}]}
      ],
-     [{"A", "1", [{"D",[{"J",[]}]},
-                  {"E",[{"I","1",[]}]}]},
-      {"B", "1", [{"F",[]},
+     [{"A", "1.0.0", [{"D",[{"J",[]}]},
+                  {"E",[{"I","1.0.0",[]}]}]},
+      {"B", "1.0.0", [{"F",[]},
                   {"G",[]}]},
-      {"C", "2", [{"H",[]}]}
+      {"C", "2.0.0", [{"H",[]}]}
      ],
      ["C"],
-     {"B", [{"A","1"}, "D", "J", "E",
-            {"B","1"}, "F", "G",
-            {"C","1"}, "H", {"I","2"}]}};
+     {"B", [{"A","1.0.0"}, "D", "J", "E",
+            {"B","1.0.0"}, "F", "G",
+            {"C","1.0.0"}, "H", {"I","2.0.0"}]}};
 upgrades(tree_c) ->
-    {[{"A", "1", [{"D",[{"J",[]}]},
-                  {"E",[{"I","1",[]}]}]},
-      {"B", "1", [{"F",[]},
+    {[{"A", "1.0.0", [{"D",[{"J",[]}]},
+                  {"E",[{"I","1.0.0",[]}]}]},
+      {"B", "1.0.0", [{"F",[]},
                   {"G",[]}]},
-      {"C", "1", [{"H",[]},
-                  {"I","2",[]}]}
+      {"C", "1.0.0", [{"H",[]},
+                  {"I","2.0.0",[]}]}
      ],
-     [{"A", "1", [{"D",[{"J",[]}]},
-                  {"E",[{"I","1",[]}]}]},
-      {"B", "1", [{"F",[]},
+     [{"A", "1.0.0", [{"D",[{"J",[]}]},
+                  {"E",[{"I","1.0.0",[]}]}]},
+      {"B", "1.0.0", [{"F",[]},
                   {"G",[]}]},
-      {"C", "1", [{"H",[]}]}
+      {"C", "1.0.0", [{"H",[]}]}
      ],
      ["C","I"],
-     {"C", [{"A","1"}, "D", "J", "E", {"I","1"},
-            {"B","1"}, "F", "G",
-            {"C","1"}, "H"]}};
+     {"C", [{"A","1.0.0"}, "D", "J", "E", {"I","1.0.0"},
+            {"B","1.0.0"}, "F", "G",
+            {"C","1.0.0"}, "H"]}};
 upgrades(tree_c2) ->
-    {[{"A", "1", [{"D",[{"J",[]}]},
-                  {"E",[{"I","1",[]}]}]},
-      {"B", "1", [{"F",[]},
+    {[{"A", "1.0.0", [{"D",[{"J",[]}]},
+                  {"E",[{"I","1.0.0",[]}]}]},
+      {"B", "1.0.0", [{"F",[]},
                   {"G",[]}]},
-      {"C", "1", [{"H",[]},
-                  {"I","2",[]}]}
+      {"C", "1.0.0", [{"H",[]},
+                  {"I","2.0.0",[]}]}
      ],
-     [{"A", "1", [{"D",[{"J",[]}]},
-                  {"E",[{"I","1",[]}]}]},
-      {"B", "1", [{"F",[]},
+     [{"A", "1.0.0", [{"D",[{"J",[]}]},
+                  {"E",[{"I","1.0.0",[]}]}]},
+      {"B", "1.0.0", [{"F",[]},
                   {"G",[]}]},
-      {"C", "1", [{"H",[{"K",[]}]},
-                  {"I","2",[]}]}
+      {"C", "1.0.0", [{"H",[{"K",[]}]},
+                  {"I","2.0.0",[]}]}
      ],
      ["C", "H"],
-     {"C", [{"A","1"}, "D", "J", "E",
-            {"B","1"}, "F", "G",
-            {"C","1"}, "H", {"I", "2"}, "K"]}};
+     {"C", [{"A","1.0.0"}, "D", "J", "E",
+            {"B","1.0.0"}, "F", "G",
+            {"C","1.0.0"}, "H", {"I", "2.0.0"}, "K"]}};
 upgrades(tree_cj) ->
-    {[{"A", "1", [{"D",[{"J", "1",[]}]},
-                  {"E",[{"I","1",[]}]}]},
-      {"B", "1", [{"F",[]},
+    {[{"A", "1.0.0", [{"D",[{"J", "1.0.0",[]}]},
+                  {"E",[{"I","1.0.0",[]}]}]},
+      {"B", "1.0.0", [{"F",[]},
                   {"G",[]}]},
-      {"C", "1", [{"H",[]},
-                  {"I","1",[]}]}
+      {"C", "1.0.0", [{"H",[]},
+                  {"I","1.0.0",[]}]}
      ],
-     [{"A", "1", [{"D",[{"J", "2", []}]},
-                  {"E",[{"I","1",[]}]}]},
-      {"B", "1", [{"F",[]},
+     [{"A", "1.0.0", [{"D",[{"J", "2.0.0", []}]},
+                  {"E",[{"I","1.0.0",[]}]}]},
+      {"B", "1.0.0", [{"F",[]},
                   {"G",[]}]},
-      {"C", "1", [{"H",[]},
-                  {"I","1",[]}]}
+      {"C", "1.0.0", [{"H",[]},
+                  {"I","1.0.0",[]}]}
      ],
      ["C","J"],
-     {"C", [{"A","1"}, "D", {"J", "1"}, "E", {"I","1"},
-            {"B","1"}, "F", "G",
-            {"C","1"}, "H"]}};
+     {"C", [{"A","1.0.0"}, "D", {"J", "1.0.0"}, "E", {"I","1.0.0"},
+            {"B","1.0.0"}, "F", "G",
+            {"C","1.0.0"}, "H"]}};
 upgrades(tree_ac) ->
-    {[{"A", "1", [{"D",[{"J",[]}]},
-                  {"E",[{"I","1",[]}]}]},
-      {"B", "1", [{"F",[]},
+    {[{"A", "1.0.0", [{"D",[{"J",[]}]},
+                  {"E",[{"I","1.0.0",[]}]}]},
+      {"B", "1.0.0", [{"F",[]},
                   {"G",[]}]},
-      {"C", "1", [{"H",[]},
-                  {"I","2",[]}]}
+      {"C", "1.0.0", [{"H",[]},
+                  {"I","2.0.0",[]}]}
      ],
-     [{"A", "1", [{"D",[{"J",[]}]},
-                  {"E",[{"I","1",[]}]}]},
-      {"B", "1", [{"F",[]},
+     [{"A", "1.0.0", [{"D",[{"J",[]}]},
+                  {"E",[{"I","1.0.0",[]}]}]},
+      {"B", "1.0.0", [{"F",[]},
                   {"G",[]}]},
-      {"C", "1", [{"H",[]}]}
+      {"C", "1.0.0", [{"H",[]}]}
      ],
      ["C","I"],
-     {"C, A", [{"A","1"}, "D", "J", "E", {"I","1"},
-               {"B","1"}, "F", "G",
-               {"C","1"}, "H"]}};
+     {"C, A", [{"A","1.0.0"}, "D", "J", "E", {"I","1.0.0"},
+               {"B","1.0.0"}, "F", "G",
+               {"C","1.0.0"}, "H"]}};
 upgrades(tree_all) ->
-    {[{"A", "1", [{"D",[{"J",[]}]},
-                  {"E",[{"I","1",[]}]}]},
-      {"B", "1", [{"F",[]},
+    {[{"A", "1.0.0", [{"D",[{"J",[]}]},
+                  {"E",[{"I","1.0.0",[]}]}]},
+      {"B", "1.0.0", [{"F",[]},
                   {"G",[]}]},
-      {"C", "1", [{"H",[]},
-                  {"I","2",[]}]}
+      {"C", "1.0.0", [{"H",[]},
+                  {"I","2.0.0",[]}]}
      ],
-     [{"A", "1", [{"D",[{"J",[]}]},
-                  {"E",[{"I","1",[]}]}]},
-      {"B", "1", [{"F",[]},
+     [{"A", "1.0.0", [{"D",[{"J",[]}]},
+                  {"E",[{"I","1.0.0",[]}]}]},
+      {"B", "1.0.0", [{"F",[]},
                   {"G",[]}]},
-      {"C", "1", [{"H",[]}]}
+      {"C", "1.0.0", [{"H",[]}]}
      ],
      ["C","I"],
-     {"", [{"A","1"}, "D", "J", "E", {"I","1"},
-           {"B","1"}, "F", "G",
-           {"C","1"}, "H"]}};
+     {"", [{"A","1.0.0"}, "D", "J", "E", {"I","1.0.0"},
+           {"B","1.0.0"}, "F", "G",
+           {"C","1.0.0"}, "H"]}};
 upgrades(delete_d) ->
-    {[{"A", "1", [{"B", [{"D", "1", []}]},
-                  {"C", [{"D", "2", []}]}]}
+    {[{"A", "1.0.0", [{"B", [{"D", "1.0.0", []}]},
+                  {"C", [{"D", "2.0.0", []}]}]}
      ],
-     [{"A", "2", [{"B", []},
+     [{"A", "2.0.0", [{"B", []},
                   {"C", []}]}
      ],
      ["A","B", "C"],
      %% upgrade vs. new tree
-     {"", [{"A","2"}, "B", "C"]}};
+     {"", [{"A","2.0.0"}, "B", "C"]}};
 upgrades(promote) ->
-    {[{"A", "1", [{"C", "1", []}]},
-      {"B", "1", [{"D", "1", []}]}
+    {[{"A", "1.0.0", [{"C", "1.0.0", []}]},
+      {"B", "1.0.0", [{"D", "1.0.0", []}]}
      ],
-     [{"A", "2", [{"C", "2", []}]},
-      {"B", "2", [{"D", "2", []}]},
-      {"C", "3", []}
+     [{"A", "2.0.0", [{"C", "2.0.0", []}]},
+      {"B", "2.0.0", [{"D", "2.0.0", []}]},
+      {"C", "3.0.0", []}
      ],
      ["A","B","C","D"],
-     {"C", [{"A","1"},{"C","3"},{"B","1"},{"D","1"}]}};
+     {"C", [{"A","1.0.0"},{"C","3.0.0"},{"B","1.0.0"},{"D","1.0.0"}]}};
 upgrades(stable_lock) ->
-    {[{"A", "1", [{"C", "1", []}]},
-      {"B", "1", [{"D", "1", []}]}
+    {[{"A", "1.0.0", [{"C", "1.0.0", []}]},
+      {"B", "1.0.0", [{"D", "1.0.0", []}]}
      ], % lock after this
-     [{"A", "2", [{"C", "2", []}]},
-      {"B", "2", [{"D", "2", []}]}
+     [{"A", "2.0.0", [{"C", "2.0.0", []}]},
+      {"B", "2.0.0", [{"D", "2.0.0", []}]}
      ],
      [],
      %% Run a regular lock and no app should be upgraded
-     {"any", [{"A","1"},{"C","1"},{"B","1"},{"D","1"}]}};
+     {"any", [{"A","1.0.0"},{"C","1.0.0"},{"B","1.0.0"},{"D","1.0.0"}]}};
 upgrades(fwd_lock) ->
-    {[{"A", "1", [{"C", "1", []}]},
-      {"B", "1", [{"D", "1", []}]}
+    {[{"A", "1.0.0", [{"C", "1.0.0", []}]},
+      {"B", "1.0.0", [{"D", "1.0.0", []}]}
      ],
-     [{"A", "2", [{"C", "2", []}]},
-      {"B", "2", [{"D", "2", []}]}
+     [{"A", "2.0.0", [{"C", "2.0.0", []}]},
+      {"B", "2.0.0", [{"D", "2.0.0", []}]}
      ],
      ["A","B","C","D"],
      %% For this one, we should build, rewrite the lock
      %% file to include the result post-upgrade, and then
      %% run a regular lock to see that the lock file is respected
      %% in deps.
-     {"any", [{"A","2"},{"C","2"},{"B","2"},{"D","2"}]}};
+     {"any", [{"A","2.0.0"},{"C","2.0.0"},{"B","2.0.0"},{"D","2.0.0"}]}};
 upgrades(compile_upgrade_parity) ->
-    {[{"A", "1", [{"D",[{"J",[]}]},
-                  {"E",[{"I","1",[]}]}]},
-      {"B", "1", [{"F",[]},
+    {[{"A", "1.0.0", [{"D",[{"J",[]}]},
+                  {"E",[{"I","1.0.0",[]}]}]},
+      {"B", "1.0.0", [{"F",[]},
                   {"G",[]}]},
-      {"C", "1", [{"H",[]},
-                  {"I","2",[]}]}
+      {"C", "1.0.0", [{"H",[]},
+                  {"I","2.0.0",[]}]}
      ],
      [],
      [],
-     {"", [{"A","1"}, "D", "J", "E", {"I","1"},
-           {"B","1"}, "F", "G",
-           {"C","1"}, "H"]}};
+     {"", [{"A","1.0.0"}, "D", "J", "E", {"I","1.0.0"},
+           {"B","1.0.0"}, "F", "G",
+           {"C","1.0.0"}, "H"]}};
 upgrades(umbrella_config) ->
-    {[{"A", "1", []}],
-     [{"A", "2", []}],
+    {[{"A", "1.0.0", []}],
+     [{"A", "2.0.0", []}],
      ["A"],
-     {"A", [{"A","2"}]}};
+     {"A", [{"A","2.0.0"}]}};
 upgrades(profiles) ->
     %% Ensure that we can unlock deps under a given profile;
     %% B and C should both be in a custom profile
     %% and must not be locked.
-    {[{"A", "1", [{"D",[]},
-                  {"E","3",[]}]},
-      {"B", "1", [{"F","1",[]},
+    {[{"A", "1.0.0", [{"D",[]},
+                  {"E","3.0.0",[]}]},
+      {"B", "1.0.0", [{"F","1.0.0",[]},
                   {"G",[]}]},
-      {"C", "0", [{"H","3",[]},
+      {"C", "0.0.0", [{"H","3.0.0",[]},
                   {"I",[]}]}],
-     [{"A", "2", [{"D",[]},
-                  {"E","2",[]}]},
-      {"B", "2", [{"F","2",[]},
+     [{"A", "2.0.0", [{"D",[]},
+                  {"E","2.0.0",[]}]},
+      {"B", "2.0.0", [{"F","2.0.0",[]},
                   {"G",[]}]},
-      {"C", "1", [{"H","4",[]},
+      {"C", "1.0.0", [{"H","4.0.0",[]},
                   {"I",[]}]}],
      ["A","B","C","E","F","H"],
-     {"C", [{"A","1"}, "D", {"E","3"},
-            {"B","2"}, {"F","2"}, "G",
-            {"C","1"}, {"H","4"}, "I"]}};
+     {"C", [{"A","1.0.0"}, "D", {"E","3.0.0"},
+            {"B","2.0.0"}, {"F","2.0.0"}, "G",
+            {"C","1.0.0"}, {"H","4.0.0"}, "I"]}};
 upgrades(profiles_exclusion) ->
     %% Ensure that we can unlock deps under a given profile;
     %% B and C should both be in a custom profile
     %% and must not be locked.
-    {[{"A", "1", [{"D",[]},
-                  {"E","3",[]}]},
-      {"B", "1", [{"F","1",[]},
+    {[{"A", "1.0.0", [{"D",[]},
+                  {"E","3.0.0",[]}]},
+      {"B", "1.0.0", [{"F","1.0.0",[]},
                   {"G",[]}]},
-      {"C", "0", [{"H","3",[]},
+      {"C", "0.0.0", [{"H","3.0.0",[]},
                   {"I",[]}]}],
-     [{"A", "2", [{"D",[]},
-                  {"E","2",[]}]},
-      {"B", "2", [{"F","2",[]},
+     [{"A", "2.0.0", [{"D",[]},
+                  {"E","2.0.0",[]}]},
+      {"B", "2.0.0", [{"F","2.0.0",[]},
                   {"G",[]}]},
-      {"C", "1", [{"H","4",[]},
+      {"C", "1.0.0", [{"H","4.0.0",[]},
                   {"I",[]}]}],
      ["A","B","C","E","F","H"],
-     {"A", [{"A","1"}, "D", {"E","3"},
-            {"B","2"}, {"F","2"}, "G",
-            {"C","1"}, {"H","4"}, "I"]}}.
+     {"A", [{"A","1.0.0"}, "D", {"E","3.0.0"},
+            {"B","2.0.0"}, {"F","2.0.0"}, "G",
+            {"C","1.0.0"}, {"H","4.0.0"}, "I"]}}.
 
 %% TODO: add a test that verifies that unlocking files and then
 %% running the upgrade code is enough to properly upgrade things.