Merge pull request #784 from tsloughter/extract

add 'local install' and 'local upgrade'

README.md

@@ -45,6 +45,7 @@ locations ([hex.pm](http://hex.pm), git, hg, and so on).
 | tar        | Package release into tarball |
 | tree       | Print dependency tree |
 | unlock     | Unlock dependencies |
+| unstable   | Namespace providing commands that are still in flux |
 | update     | Update package index |
 | upgrade    | Fetch latest version of dep |
 | version    | Print current version of Erlang/OTP and rebar |

priv/shell-completion/bash/rebar3

@@ -33,6 +33,7 @@ _rebar3()
             tar \
             tree \
             unlock \
+            unstable \
             update \
             upgrade \
             version \
@@ -187,6 +188,8 @@ _rebar3()
     elif [[ ${prev} == tree ]] ; then
         sopts="-v"
         lopts="--verbose"
+    elif [[ ${prev} == unstable ]] ; then
+        :
     elif [[ ${prev} == update ]] ; then
         :
     elif [[ ${prev} == upgrade ]] ; then

priv/shell-completion/fish/rebar3.fish

@@ -53,6 +53,7 @@ end
 ## tar               Tar archive of release built of project.
 ## tree              Print dependency tree.
 ## unlock            Unlock dependencies.
+## unstable          Namespace providing commands that are still in flux.
 ## update            Update package index.
 ## upgrade           Upgrade dependencies.
 ## version           Print version for rebar and current Erlang.
@@ -101,7 +102,7 @@ complete -f -c 'rebar3' -n '__fish_rebar3_using_command dialyzer' -s u -l update
 complete -f -c 'rebar3' -n '__fish_rebar3_using_command dialyzer' -s s -l succ-typings -d "Enable success typing analysis. Default: true"
 
 complete -f -c 'rebar3' -n '__fish_rebar3_needs_command' -a do -d "Higher order provider for running multiple tasks in a sequence."
-complete -f -c 'rebar3' -n '__fish_rebar3_using_command do' -a 'compile, clean, ct, cover, deps, dialyzer, edoc, eunit, help, new, pkgs, release, relup, report, shell, tar, unlock, update, upgrade, version, xref,'
+complete -f -c 'rebar3' -n '__fish_rebar3_using_command do' -a 'compile, clean, ct, cover, deps, dialyzer, edoc, eunit, help, new, pkgs, release, relup, report, shell, tar, unlock, unstable, update, upgrade, version, xref,'
 ## TODO: do should understand plugins, but now it does not.
 
 complete -f -c 'rebar3' -n '__fish_rebar3_needs_command' -a edoc -d "Generate documentation using edoc."
@@ -121,14 +122,14 @@ complete -f -c 'rebar3' -n '__fish_rebar3_using_command new' -a help -d "Display
 
 complete -f -c 'rebar3' -n '__fish_rebar3_needs_command' -a paths -d "Print paths to build dirs in current profile."
 complete -f -c 'rebar3' -n '__fish_rebar3_needs_command paths' -l app -d "Comma seperated list of applications to return paths for."
-complete -f -c 'rebar3' -n '__fish_rebar3_needs_command paths' -l base -d "Return the `base' path of the current profile."
-complete -f -c 'rebar3' -n '__fish_rebar3_needs_command paths' -l bin -d Return the `bin' path of the current profile."
-complete -f -c 'rebar3' -n '__fish_rebar3_needs_command paths' -l ebin -d "Return all `ebin' paths of the current profile's applications."
-complete -f -c 'rebar3' -n '__fish_rebar3_needs_command paths' -l lib -d "Return the `lib' path of the current profile."
-complete -f -c 'rebar3' -n '__fish_rebar3_needs_command paths' -l priv -d "Return the `priv' path of the current profile's applications."
+complete -f -c 'rebar3' -n '__fish_rebar3_needs_command paths' -l base -d "Return the `base` path of the current profile."
+complete -f -c 'rebar3' -n '__fish_rebar3_needs_command paths' -l bin -d "Return the `bin` path of the current profile."
+complete -f -c 'rebar3' -n '__fish_rebar3_needs_command paths' -l ebin -d "Return all `ebin` paths of the current profile`s applications."
+complete -f -c 'rebar3' -n '__fish_rebar3_needs_command paths' -l lib -d "Return the `lib` path of the current profile."
+complete -f -c 'rebar3' -n '__fish_rebar3_needs_command paths' -l priv -d "Return the `priv` path of the current profile`s applications."
 complete -f -c 'rebar3' -n '__fish_rebar3_needs_command paths' -s s -l separator -d "In case of multiple return paths, the separator character to use to join them."
-complete -f -c 'rebar3' -n '__fish_rebar3_needs_command paths' -l src -d "Return the `src' path of the current profile's applications."
-complete -f -c 'rebar3' -n '__fish_rebar3_needs_command paths' -l rel -d "Return the `rel' path of the current profile."
+complete -f -c 'rebar3' -n '__fish_rebar3_needs_command paths' -l src -d "Return the `src` path of the current profile`s applications."
+complete -f -c 'rebar3' -n '__fish_rebar3_needs_command paths' -l rel -d "Return the `rel` path of the current profile."
 
 complete -f -c 'rebar3' -n '__fish_rebar3_needs_command' -a pkgs -d "List available packages."
 complete -f -c 'rebar3' -n '__fish_rebar3_needs_command' -a release -d "Build release of project."
@@ -137,8 +138,8 @@ complete -f -c 'rebar3' -n '__fish_rebar3_needs_command' -a report -d "Provide a
 
 complete -f -c 'rebar3' -n '__fish_rebar3_needs_command' -a shell -d "Run shell with project apps and deps in path."
 complete -f -c 'rebar3' -n '__fish_rebar3_using_command shell' -l config -d "Allows to load a config file, if any. Defaults to the sys_config entry defined for relx if present."
-complete -f -c 'rebar3' -n '__fish_rebar3_using_command shell' -l name -d "equivalent to erlang's -name"
-complete -f -c 'rebar3' -n '__fish_rebar3_using_command shell' -l sname -d "equivalent to erlang's -sname"
+complete -f -c 'rebar3' -n '__fish_rebar3_using_command shell' -l name -d "equivalent to erlang`s -name"
+complete -f -c 'rebar3' -n '__fish_rebar3_using_command shell' -l sname -d "equivalent to erlang`s -sname"
 
 complete -f -c 'rebar3' -n '__fish_rebar3_needs_command' -a tar -d "Tar archive of release built of project."
 complete -f -c 'rebar3' -n '__fish_rebar3_using_command tar' -s n -l relname -d "Specify the name for the release that will be generated"
@@ -168,6 +169,10 @@ complete -f -c 'rebar3' -n '__fish_rebar3_needs_command tree' -s v -l verbose  -
 
 complete -f -c 'rebar3' -n '__fish_rebar3_needs_command' -a unlock -d "Unlock dependencies."
 
+complete -f -c 'rebar3' -n '__fish_rebar3_needs_command' -a unstable -d "Namespace providing commands that are still in flux."
+
+complete -f -c 'rebar3' -n '__fish_rebar3_using_command unstable' -a 'install upgrade'
+
 complete -f -c 'rebar3' -n '__fish_rebar3_needs_command' -a update -d "Update package index."
 
 complete -f -c 'rebar3' -n '__fish_rebar3_needs_command' -a upgrade -d "Upgrade dependencies."

priv/shell-completion/zsh/_rebar3

@@ -111,11 +111,11 @@ _rebar3 () {
                 (path)
                     _arguments \
                         '(--app)--app[Comma seperated list of applications to return paths for.]:apps' \
-                        '(--base)--base[Return the `base' path of the current profile.]' \
-                        '(--bin)--bin[Return the `bin' path of the current profile.]' \
-                        '(--ebin)--ebin[Return all `ebin' paths of the current profile's applications.]' \
-                        '(--lib)--lib[Return the `lib' path of the current profile.]' \
-                        '(--priv)--priv[Return the `priv' path of the current profile's applications.]' \
+                        '(--base)--base[Return the `base` path of the current profile.]' \
+                        '(--bin)--bin[Return the `bin` path of the current profile.]' \
+                        '(--ebin)--ebin[Return all `ebin` paths of the current profile`s applications.]' \
+                        '(--lib)--lib[Return the `lib` path of the current profile.]' \
+                        '(--priv)--priv[Return the `priv` path of the current profile`s applications.]' \
                         '(-s --separator)--separator[In case of multiple return paths, the separator character to use to join them.]' \
                     && ret=0
                 ;;
@@ -212,6 +212,11 @@ _rebar3 () {
                         '*: :_rebar3_list_deps' \
                     && ret=0
                 ;;
+                (unstable)
+                    _arguments \
+                        '*: :(install upgrade)' \
+                    && ret=0
+                ;;
                 (update)
                     _message 'rebar update' && ret=0
                 ;;
@@ -254,6 +259,7 @@ _rebar3_tasks() {
         'tar:Tar archive of release built of project.'
         'tree:Print dependency tree.'
         'unlock:Unlock dependencies.'
+        'unstable:Namespace providing commands that are still in flux.'
         'update:Update package index.'
         'upgrade:Upgrade dependencies.'
         'version:Print version for rebar and current Erlang.'

src/rebar.app.src

@@ -23,7 +23,7 @@
                   erlware_commons,
                   providers,
                   bbmustache,
-          ssl_verify_hostname,
+                  ssl_verify_hostname,
                   relx,
                   inets]},
   {env, [
@@ -49,6 +49,8 @@
                      rebar_prv_eunit,
                      rebar_prv_help,
                      rebar_prv_install_deps,
+                     rebar_prv_local_install,
+                     rebar_prv_local_upgrade,
                      rebar_prv_lock,
                      rebar_prv_new,
                      rebar_prv_packages,

src/rebar3.erl

@@ -26,7 +26,8 @@
 %% -------------------------------------------------------------------
 -module(rebar3).
 
--export([main/1,
+-export([main/0,
+         main/1,
          run/1,
          run/2,
          global_option_spec_list/0,
@@ -42,6 +43,12 @@
 %% Public API
 %% ====================================================================
 
+%% For running with:
+%% erl +sbtu +A0 -noinput -mode minimal -boot start_clean -s rebar3 main -extra "$@"
+main() ->
+    List = init:get_plain_arguments(),
+    main(List).
+
 %% escript Entry point
 -spec main(list()) -> no_return().
 main(Args) ->
@@ -146,7 +153,14 @@ init_config() ->
     %% resources out of the escript
     State1 = try
                  ScriptName = filename:absname(escript:script_name()),
-                 rebar_state:escript_path(State, ScriptName)
+                 %% Running with 'erl -s rebar3 main' still sets a name for some reason
+                 %% so verify it is a real file
+                 case filelib:is_regular(ScriptName) of
+                     true ->
+                         rebar_state:escript_path(State, ScriptName);
+                     false ->
+                         State
+                 end
              catch
                  _:_ ->
                      State

src/rebar_pkg_resource.erl

@@ -9,7 +9,8 @@
         ,needs_update/2
         ,make_vsn/1]).
 
--export([ssl_opts/1]).
+-export([request/2
+        ,ssl_opts/1]).
 
 -include("rebar.hrl").
 -include_lib("public_key/include/OTP-PUB-KEY.hrl").

src/rebar_prv_local_install.erl

@@ -0,0 +1,96 @@
+%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*-
+%% ex: ts=4 sw=4 et
+
+-module(rebar_prv_local_install).
+
+-behaviour(provider).
+
+-export([init/1,
+         do/1,
+         format_error/1]).
+
+-export([extract_escript/2]).
+
+-include("rebar.hrl").
+-include_lib("kernel/include/file.hrl").
+
+-define(PROVIDER, install).
+-define(NAMESPACE, unstable).
+-define(DEPS, []).
+
+%% ===================================================================
+%% Public API
+%% ===================================================================
+
+-spec init(rebar_state:t()) -> {ok, rebar_state:t()}.
+init(State) ->
+    State1 =
+        rebar_state:add_provider(State,
+                                providers:create([{name, ?PROVIDER},
+                                                  {module, ?MODULE},
+                                                  {bare, true},
+                                                  {namespace, ?NAMESPACE},
+                                                  {deps, ?DEPS},
+                                                  {example, "rebar3 unstable install"},
+                                                  {short_desc, "Extract libs from rebar3 escript along with a run script."},
+                                                  {desc, ""},
+                                                  {opts, []}])),
+    {ok, State1}.
+
+-spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}.
+do(State) ->
+    case os:type() of
+        {win32, _} ->
+            ?ERROR("Sorry, this feature is not yet available on Windows.", []),
+            {ok, State};
+        _ ->
+            case rebar_state:escript_path(State) of
+                undefined ->
+                    ?INFO("Already running from an unpacked rebar3. Nothing to do...", []),
+                    {ok, State};
+                ScriptPath ->
+                    extract_escript(State, ScriptPath)
+            end
+    end.
+
+-spec format_error(any()) -> iolist().
+format_error(Reason) ->
+    io_lib:format("~p", [Reason]).
+
+bin_contents(OutputDir) ->
+    <<"
+#!/usr/bin/env sh
+
+erl -pa ", (ec_cnv:to_binary(OutputDir))/binary,"/*/ebin +sbtu +A0 -noinput -boot start_clean -s rebar3 main -extra \"$@\"
+">>.
+
+extract_escript(State, ScriptPath) ->
+    {ok, Escript} = escript:extract(ScriptPath, []),
+    {archive, Archive} = lists:keyfind(archive, 1, Escript),
+
+    %% Extract contents of Archive to ~/.cache/rebar3/lib
+    %% And add a rebar3 bin script to ~/.cache/rebar3/bin
+    Opts = rebar_state:opts(State),
+    OutputDir = filename:join(rebar_dir:global_cache_dir(Opts), "lib"),
+    filelib:ensure_dir(filename:join(OutputDir, "empty")),
+
+    ?INFO("Extracting rebar3 libs to ~s...", [OutputDir]),
+    zip:extract(Archive, [{cwd, OutputDir}]),
+
+    BinDir = filename:join(rebar_dir:global_cache_dir(Opts), "bin"),
+    BinFile = filename:join(BinDir, "rebar3"),
+    filelib:ensure_dir(BinFile),
+
+    {ok, #file_info{mode = _,
+                    uid = Uid,
+                    gid = Gid}} = file:read_file_info(ScriptPath, [mode, uid, gid]),
+
+    ?INFO("Writing rebar3 run script ~s...", [BinFile]),
+    file:write_file(BinFile, bin_contents(OutputDir)),
+    ok = file:write_file_info(BinFile, #file_info{mode=33277,
+                                                  uid=Uid,
+                                                  gid=Gid}),
+
+    ?INFO("Add to $PATH for use: export PATH=$PATH:~s", [BinDir]),
+
+    {ok, State}.

src/rebar_prv_local_upgrade.erl

@@ -0,0 +1,84 @@
+%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*-
+%% ex: ts=4 sw=4 et
+
+-module(rebar_prv_local_upgrade).
+
+-behaviour(provider).
+
+-export([init/1,
+         do/1,
+         format_error/1]).
+
+-include("rebar.hrl").
+-include_lib("kernel/include/file.hrl").
+
+-define(PROVIDER, upgrade).
+-define(NAMESPACE, unstable).
+-define(DEPS, []).
+
+%% ===================================================================
+%% Public API
+%% ===================================================================
+
+-spec init(rebar_state:t()) -> {ok, rebar_state:t()}.
+init(State) ->
+    State1 =
+        rebar_state:add_provider(State,
+                                providers:create([{name, ?PROVIDER},
+                                                  {module, ?MODULE},
+                                                  {bare, true},
+                                                  {namespace, ?NAMESPACE},
+                                                  {deps, ?DEPS},
+                                                  {example, "rebar3 unstable upgrade"},
+                                                  {short_desc, "Download latest rebar3 escript and extract."},
+                                                  {desc, ""},
+                                                  {opts, []}])),
+    {ok, State1}.
+
+-spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}.
+do(State) ->
+    case os:type() of
+        {win32, _} ->
+            ?ERROR("Sorry, this feature is not yet available on Windows.", []),
+            {ok, State};
+        _ ->
+            Md5 = case rebar_state:escript_path(State) of
+                      undefined ->
+                          false;
+                      ScriptPath ->
+                          get_md5(ScriptPath)
+                  end,
+
+            case maybe_fetch_rebar3(Md5) of
+                {saved, TmpRebar3} ->
+                    rebar_prv_local_install:extract_escript(State, TmpRebar3);
+                _ ->
+                    {ok, State}
+            end
+    end.
+
+-spec format_error(any()) -> iolist().
+format_error(Reason) ->
+    io_lib:format("~p", [Reason]).
+
+%% Internal
+
+get_md5(Rebar3Path) ->
+    {ok, Rebar3File} = file:read_file(Rebar3Path),
+    Digest = crypto:hash(md5, Rebar3File),
+    DigestHex = lists:flatten([io_lib:format("~2.16.0B", [X]) || X <- binary_to_list(Digest)]),
+    string:to_lower(DigestHex).
+
+maybe_fetch_rebar3(Rebar3Md5) ->
+    TmpDir = ec_file:insecure_mkdtemp(),
+    TmpFile = filename:join(TmpDir, "rebar3"),
+    case rebar_pkg_resource:request("https://s3.amazonaws.com/rebar3/rebar4", Rebar3Md5) of
+        {ok, Binary, _ETag} ->
+            file:write_file(TmpFile, Binary),
+            {saved, TmpFile};
+        error ->
+            ?ERROR("Unable to fetch latest rebar3 escript. Please try again later.", []);
+        _ ->
+            ?CONSOLE("No upgrade available", []),
+            up_to_date
+    end.