From c9fbb4a44feac75a325c5831293901d5cb12075a Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Thu, 18 Oct 2018 15:38:48 -0400 Subject: [PATCH 1/9] Fix shell hook expansion on windows Dollar Variable expansion (`$VAR`) was inadvertently disabled for windows variables, although %VARIABLES% already worked. This reduced the portability of hooks in general. Additionally, tests would fail on windows due to bad quoting of paths: the path C:/a/b/c would fail when passed to the command `cmd /q /c C:/a/b/c` because it would interpret /a /b and /c as 3 options. Using quotes makes the tests pass. --- src/rebar_utils.erl | 4 ++-- test/rebar_hooks_SUITE.erl | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/rebar_utils.erl b/src/rebar_utils.erl index 1769b797..11add61f 100644 --- a/src/rebar_utils.erl +++ b/src/rebar_utils.erl @@ -156,7 +156,7 @@ sh_send(Command0, String, Options0) -> Options = [expand_sh_flag(V) || V <- proplists:compact(Options0 ++ DefaultOptions)], - Command = lists:flatten(patch_on_windows(Command0, proplists:get_value(env, Options, []))), + Command = lists:flatten(patch_on_windows(Command0, proplists:get_value(env, Options0, []))), PortSettings = proplists:get_all_values(port_settings, Options) ++ [exit_status, {line, 16384}, use_stdio, stderr_to_stdout, hide], Port = open_port({spawn, Command}, PortSettings), @@ -187,7 +187,7 @@ sh(Command0, Options0) -> ErrorHandler = proplists:get_value(error_handler, Options), OutputHandler = proplists:get_value(output_handler, Options), - Command = lists:flatten(patch_on_windows(Command0, proplists:get_value(env, Options, []))), + Command = lists:flatten(patch_on_windows(Command0, proplists:get_value(env, Options0, []))), PortSettings = proplists:get_all_values(port_settings, Options) ++ [exit_status, {line, 16384}, use_stdio, stderr_to_stdout, hide, eof], ?DEBUG("Port Cmd: ~ts\nPort Opts: ~p\n", [Command, PortSettings]), diff --git a/test/rebar_hooks_SUITE.erl b/test/rebar_hooks_SUITE.erl index aae7ea0c..29e343f0 100644 --- a/test/rebar_hooks_SUITE.erl +++ b/test/rebar_hooks_SUITE.erl @@ -84,7 +84,7 @@ run_hooks_once(Config) -> Name = rebar_test_utils:create_random_name("app1_"), Vsn = rebar_test_utils:create_random_vsn(), - RebarConfig = [{pre_hooks, [{compile, "mkdir $REBAR_ROOT_DIR/blah"}]}], + RebarConfig = [{pre_hooks, [{compile, "mkdir \"$REBAR_ROOT_DIR/blah\""}]}], rebar_test_utils:create_config(AppDir, RebarConfig), rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), rebar_test_utils:run_and_check(Config, RebarConfig, ["compile"], {ok, [{app, Name, valid}]}). @@ -96,7 +96,7 @@ run_hooks_once_profiles(Config) -> Name = rebar_test_utils:create_random_name("app1_"), Vsn = rebar_test_utils:create_random_vsn(), - RebarConfig = [{profiles, [{hooks, [{pre_hooks, [{compile, "mkdir $REBAR_ROOT_DIR/blah"}]}]}]}], + RebarConfig = [{profiles, [{hooks, [{pre_hooks, [{compile, "mkdir \"$REBAR_ROOT_DIR/blah\""}]}]}]}], rebar_test_utils:create_config(AppDir, RebarConfig), rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), rebar_test_utils:run_and_check(Config, RebarConfig, ["as", "hooks", "compile"], {ok, [{app, Name, valid}]}). @@ -236,7 +236,7 @@ root_hooks(Config) -> rebar_test_utils:create_app(SubAppsDir, Name, Vsn, [kernel, stdlib]), rebar_test_utils:create_config(SubAppsDir, [{provider_hooks, [{post, [{compile, clean}]}]}]), - RConfFile = rebar_test_utils:create_config(AppDir, [{pre_hooks, [{compile, "mkdir $REBAR_ROOT_DIR/blah"}]}]), + RConfFile = rebar_test_utils:create_config(AppDir, [{pre_hooks, [{compile, "mkdir \"$REBAR_ROOT_DIR/blah\""}]}]), {ok, RConf} = file:consult(RConfFile), %% Build with deps. From bf039fa4c4d7ea9dc0d9a842f2ab2aa93a9c61d3 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Sat, 20 Oct 2018 09:54:51 -0400 Subject: [PATCH 2/9] Fallback when logging isn't initialized In some cases, such as when the global rebar.config file contains typoes and invalid terms, the rebar3 executable fails when trying to log the error since it hasn't been set yet, such as in #1792 This patch fixes that by going for a fallback mechanism. --- src/rebar_log.erl | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/rebar_log.erl b/src/rebar_log.erl index 91503461..7fc2312f 100644 --- a/src/rebar_log.erl +++ b/src/rebar_log.erl @@ -93,11 +93,18 @@ get_level() -> end. log(Level = error, Str, Args) -> - {ok, LogState} = application:get_env(rebar, log), - ec_cmd_log:Level(LogState, lists:flatten(cf:format("~!^~ts~n", [Str])), Args); + case application:get_env(rebar, log) of + {ok, LogState} -> + NewStr = lists:flatten(cf:format("~!^~ts~n", [Str])), + ec_cmd_log:Level( LogState, NewStr, Args); + undefined -> % fallback + io:format(standard_error, Str++"~n", Args) + end; log(Level, Str, Args) -> - {ok, LogState} = application:get_env(rebar, log), - ec_cmd_log:Level(LogState, Str++"~n", Args). + case application:get_env(rebar, log) of + {ok, LogState} -> ec_cmd_log:Level(LogState, Str++"~n", Args); + undefined -> io:format(Str++"~n", Args) + end. crashdump(Str, Args) -> crashdump("rebar3.crashdump", Str, Args). From a279020d1b860313719a3ba07972004b8235146d Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Sat, 20 Oct 2018 16:14:03 -0400 Subject: [PATCH 3/9] check if git/hg is installed This PR is a simpler and mergeable version of #1853 by @shamis, since the provider changed format enough to make merging difficult. Compared to #1853, this also puts the responsibility on each resource to check rather than adding a new optional callback. The process dictionary is use as a warning/check cache. --- src/rebar_git_resource.erl | 19 +++++++++++++++++++ src/rebar_hg_resource.erl | 19 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/src/rebar_git_resource.erl b/src/rebar_git_resource.erl index 29c9ad79..03aa6d84 100644 --- a/src/rebar_git_resource.erl +++ b/src/rebar_git_resource.erl @@ -21,6 +21,7 @@ init(Type, _State) -> {ok, Resource}. lock(AppInfo, _) -> + check_type_support(), lock_(rebar_app_info:dir(AppInfo), rebar_app_info:source(AppInfo)). lock_(AppDir, {git, Url, _}) -> @@ -43,6 +44,7 @@ lock_(AppDir, {git, Url}) -> %% Return true if either the git url or tag/branch/ref is not the same as the currently %% checked out git repo for the dep needs_update(AppInfo, _) -> + check_type_support(), needs_update_(rebar_app_info:dir(AppInfo), rebar_app_info:source(AppInfo)). needs_update_(Dir, {git, Url, {tag, Tag}}) -> @@ -111,6 +113,7 @@ parse_git_url(not_scp, Url) -> end. download(TmpDir, AppInfo, State, _) -> + check_type_support(), case download_(TmpDir, rebar_app_info:source(AppInfo), State) of {ok, _} -> ok; @@ -307,3 +310,19 @@ parse_tags(Dir) -> end end end. + +check_type_support() -> + case get({is_supported, ?MODULE}) of + true -> + ok; + _ -> + case rebar_utils:sh("git --version", [{return_on_error, true}, + {use_stdout, false}]) of + {error, _} -> + ?ABORT("git not installed", []); + _ -> + put({is_supported, ?MODULE}, true), + ok + end + end. + diff --git a/src/rebar_hg_resource.erl b/src/rebar_hg_resource.erl index 21d4b9d7..8139d04f 100644 --- a/src/rebar_hg_resource.erl +++ b/src/rebar_hg_resource.erl @@ -18,6 +18,7 @@ init(Type, _State) -> {ok, Resource}. lock(AppInfo, _) -> + check_type_support(), lock_(rebar_app_info:dir(AppInfo), rebar_app_info:source(AppInfo)). lock_(AppDir, {hg, Url, _}) -> @@ -61,6 +62,7 @@ needs_update_(Dir, {hg, Url, Ref}) -> not ((LocalRef =:= TargetRef) andalso compare_url(Dir, Url)). download(TmpDir, AppInfo, State, _) -> + check_type_support(), case download_(TmpDir, rebar_app_info:source(AppInfo), State) of {ok, _} -> ok; @@ -110,6 +112,7 @@ download_(Dir, {hg, Url, Rev}, _State) -> [{cd, filename:dirname(Dir)}]). make_vsn(AppInfo, _) -> + check_type_support(), make_vsn_(rebar_app_info:dir(AppInfo)). make_vsn_(Dir) -> @@ -183,3 +186,19 @@ parse_hg_url("http://" ++ HostPath) -> parse_hg_url("https://" ++ HostPath) -> [Host | Path] = rebar_string:lexemes(HostPath, "/"), {Host, filename:rootname(filename:join(Path), ".hg")}. + +check_type_support() -> + case get({is_supported, ?MODULE}) of + true -> + ok; + false -> + case rebar_utils:sh("hg --version", [{return_on_error, true}, + {use_stdout, false}]) of + {error, _} -> + ?ABORT("hg not installed", []); + _ -> + put({is_supported, ?MODULE}, true), + ok + end + end. + From 9b03dacf2b7829b584d26a999f80c315ae8ce897 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Mon, 22 Oct 2018 19:46:09 -0400 Subject: [PATCH 4/9] Allow Breakpoints during task runs This is mostly useful for tests, where a test suite of any kind can be interrupted halfway through so that the user can probe the running system to see what is happening. This is done as follows: 1. the user must call `r3:break()` in a test suite 2. the user runs the task as `r3:async_do(ct)` 3. the test holds up and the user can do whatever 4. the user calls `r3:resume()` and the test proceeds as normal A safeguard is added so that breakpoints are only triggered in the shell in async mode Sample session: $ rebar3 shell ... 1> rebar_agent:async_do(ct). ok ... Running Common Test suites... %%% rebar_alias_SUITE: . === BREAK === 2> % 2> r3:resume(). ok 3> ..... %%% rebar_as_SUITE: ........... %%% rebar_compile_SUITE: ...... ... --- src/r3.erl | 43 ++++++++++++++++++++++++++++++++++++++++++- src/rebar_agent.erl | 38 +++++++++++++++++++++++++++++++++++++- 2 files changed, 79 insertions(+), 2 deletions(-) diff --git a/src/r3.erl b/src/r3.erl index bbf9eea0..a79cc3a1 100644 --- a/src/r3.erl +++ b/src/r3.erl @@ -1,8 +1,9 @@ %%% @doc external alias for `rebar_agent' for more convenient %%% calls from a shell. -module(r3). --export([do/1, do/2]). +-export([do/1, do/2, async_do/1, async_do/2, break/0, resume/0]). -export(['$handle_undefined_function'/2]). +-include("rebar.hrl"). %% @doc alias for `rebar_agent:do/1' -spec do(atom()) -> ok | {error, term()}. @@ -12,6 +13,46 @@ do(Command) -> rebar_agent:do(Command). -spec do(atom(), atom()) -> ok | {error, term()}. do(Namespace, Command) -> rebar_agent:do(Namespace, Command). +%% @async_doc alias for `rebar_agent:async_do/1' +-spec async_do(atom()) -> ok | {error, term()}. +async_do(Command) -> rebar_agent:async_do(Command). + +%% @async_doc alias for `rebar_agent:async_do/2' +-spec async_do(atom(), atom()) -> ok | {error, term()}. +async_do(Namespace, Command) -> rebar_agent:async_do(Namespace, Command). + +break() -> + case whereis(rebar_agent) of % is the shell running + undefined -> + ok; + Pid -> + {dictionary, Dict} = process_info(Pid, dictionary), + case lists:keyfind(cmd_type, 1, Dict) of + {cmd_type, async} -> + Self = self(), + Ref = make_ref(), + spawn_link(fun() -> + register(r3_breakpoint_handler, self()), + receive + resume -> + Self ! Ref + end + end), + io:format(user, "~n=== BREAK ===~n", []), + receive + Ref -> ok + end; + _ -> + ?DEBUG("ignoring breakpoint since command is not run " + "in async mode", []), + ok + end + end. + +resume() -> + r3_breakpoint_handler ! resume, + ok. + %% @private defer to rebar_agent '$handle_undefined_function'(Cmd, Args) -> rebar_agent:'$handle_undefined_function'(Cmd, Args). diff --git a/src/rebar_agent.erl b/src/rebar_agent.erl index 445ae54d..b4734f16 100644 --- a/src/rebar_agent.erl +++ b/src/rebar_agent.erl @@ -1,7 +1,7 @@ %%% @doc Runs a process that holds a rebar3 state and can be used %%% to statefully maintain loaded project state into a running VM. -module(rebar_agent). --export([start_link/1, do/1, do/2]). +-export([start_link/1, do/1, do/2, async_do/1, async_do/2]). -export(['$handle_undefined_function'/2]). -export([init/1, handle_call/3, handle_cast/2, handle_info/2, @@ -35,6 +35,18 @@ do(Namespace, Command) when is_atom(Namespace), is_atom(Command) -> do(Namespace, Args) when is_atom(Namespace), is_list(Args) -> gen_server:call(?MODULE, {cmd, Namespace, do, Args}, infinity). +-spec async_do(atom()) -> ok | {error, term()}. +async_do(Command) when is_atom(Command) -> + gen_server:cast(?MODULE, {cmd, Command}); +async_do(Args) when is_list(Args) -> + gen_server:cast(?MODULE, {cmd, default, do, Args}). + +-spec async_do(atom(), atom()) -> ok. +async_do(Namespace, Command) when is_atom(Namespace), is_atom(Command) -> + gen_server:cast(?MODULE, {cmd, Namespace, Command}); +async_do(Namespace, Args) when is_atom(Namespace), is_list(Args) -> + gen_server:cast(?MODULE, {cmd, Namespace, do, Args}). + '$handle_undefined_function'(Cmd, [Namespace, Args]) -> gen_server:call(?MODULE, {cmd, Namespace, Cmd, Args}, infinity); '$handle_undefined_function'(Cmd, [Args]) -> @@ -54,20 +66,44 @@ init(State) -> %% @private handle_call({cmd, Command}, _From, State=#state{state=RState, cwd=Cwd}) -> MidState = maybe_show_warning(State), + put(cmd_type, sync), {Res, NewRState} = run(default, Command, "", RState, Cwd), + put(cmd_type, undefined), {reply, Res, MidState#state{state=NewRState}, hibernate}; handle_call({cmd, Namespace, Command}, _From, State = #state{state=RState, cwd=Cwd}) -> MidState = maybe_show_warning(State), + put(cmd_type, sync), {Res, NewRState} = run(Namespace, Command, "", RState, Cwd), + put(cmd_type, undefined), {reply, Res, MidState#state{state=NewRState}, hibernate}; handle_call({cmd, Namespace, Command, Args}, _From, State = #state{state=RState, cwd=Cwd}) -> MidState = maybe_show_warning(State), + put(cmd_type, sync), {Res, NewRState} = run(Namespace, Command, Args, RState, Cwd), + put(cmd_type, undefined), {reply, Res, MidState#state{state=NewRState}, hibernate}; handle_call(_Call, _From, State) -> {noreply, State}. %% @private +handle_cast({cmd, Command}, State=#state{state=RState, cwd=Cwd}) -> + MidState = maybe_show_warning(State), + put(cmd_type, async), + {_, NewRState} = run(default, Command, "", RState, Cwd), + put(cmd_type, undefined), + {noreply, MidState#state{state=NewRState}, hibernate}; +handle_cast({cmd, Namespace, Command}, State = #state{state=RState, cwd=Cwd}) -> + MidState = maybe_show_warning(State), + put(cmd_type, async), + {_, NewRState} = run(Namespace, Command, "", RState, Cwd), + put(cmd_type, undefined), + {noreply, MidState#state{state=NewRState}, hibernate}; +handle_cast({cmd, Namespace, Command, Args}, State = #state{state=RState, cwd=Cwd}) -> + MidState = maybe_show_warning(State), + put(cmd_type, async), + {_, NewRState} = run(Namespace, Command, Args, RState, Cwd), + put(cmd_type, undefined), + {noreply, MidState#state{state=NewRState}, hibernate}; handle_cast(_Cast, State) -> {noreply, State}. From f96dcbb342d585996b04bd5b6fff9cbc955440cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boroska=20Andra=CC=81s?= Date: Sat, 27 Oct 2018 22:25:57 +0200 Subject: [PATCH 5/9] Fix misleading config in app template Uncommenting the sys.config shell setting in app template rebar.config works now as expected. --- priv/templates/app_rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/priv/templates/app_rebar.config b/priv/templates/app_rebar.config index 203ce4ac..a3f5b8c6 100644 --- a/priv/templates/app_rebar.config +++ b/priv/templates/app_rebar.config @@ -2,6 +2,6 @@ {deps, []}. {shell, [ - % {config, [{config, "config/sys.config"}]}, + % {config, "config/sys.config"}, {apps, [{{name}}]} ]}. From 0f6f1630176da37af5971a6fa51f57c0733f949d Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Mon, 29 Oct 2018 11:38:00 -0600 Subject: [PATCH 6/9] set app's dir before setting app_info deps (#1928) --- src/rebar_app_discover.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/rebar_app_discover.erl b/src/rebar_app_discover.erl index 74681c73..21dea29c 100644 --- a/src/rebar_app_discover.erl +++ b/src/rebar_app_discover.erl @@ -280,7 +280,8 @@ find_apps(LibDirs, SrcDirs, Validate, State) -> -spec find_app(file:filename_all(), valid | invalid | all) -> {true, rebar_app_info:t()} | false. find_app(AppDir, Validate) -> {Config, SrcDirs} = find_config_src(AppDir, ["src"]), - AppInfo = rebar_app_info:update_opts(rebar_app_info:new(), dict:new(), Config), + AppInfo = rebar_app_info:update_opts(rebar_app_info:dir(rebar_app_info:new(), AppDir), + dict:new(), Config), find_app_(AppInfo, AppDir, SrcDirs, Validate). %% @doc check that a given app in a directory is there, and whether it's From f0826929de675d04f8cb81afa0f3e7a879984446 Mon Sep 17 00:00:00 2001 From: Niklas Johansson Date: Wed, 31 Oct 2018 19:30:18 +0100 Subject: [PATCH 7/9] templates/gitignore ignore emacs temp files emacs creates temp files that ends with tilde (~). These temp files should never be commited. It is therefore safe to ignore them. Signed-off-by: Niklas Johansson --- THANKS | 1 + priv/templates/gitignore | 1 + 2 files changed, 2 insertions(+) diff --git a/THANKS b/THANKS index d6e38696..e4bb8720 100644 --- a/THANKS +++ b/THANKS @@ -140,3 +140,4 @@ Heinz N. Gies Roberto Aloi Andrew McRobb Drew Varner +Niklas Johansson diff --git a/priv/templates/gitignore b/priv/templates/gitignore index 40ca6526..f1c45545 100644 --- a/priv/templates/gitignore +++ b/priv/templates/gitignore @@ -16,3 +16,4 @@ _build .idea *.iml rebar3.crashdump +*~ From 60dc0504bf46e9ece179da9b20c54e0e2a3e11c6 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Thu, 8 Nov 2018 14:09:07 -0700 Subject: [PATCH 8/9] fix compilation of global plugins (#1935) --- src/rebar3.erl | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/rebar3.erl b/src/rebar3.erl index 059d5302..a490a157 100644 --- a/src/rebar3.erl +++ b/src/rebar3.erl @@ -402,18 +402,21 @@ state_from_global_config(Config, GlobalConfigFile) -> Resources = application:get_env(rebar, resources, []), GlobalConfigThrowAway = rebar_state:create_resources(Resources, GlobalConfigThrowAway0), - GlobalState = case rebar_state:get(GlobalConfigThrowAway, plugins, []) of + Compilers = application:get_env(rebar, compilers, []), + GlobalConfigThrowAway1 = rebar_state:compilers(GlobalConfigThrowAway, Compilers), + + GlobalState = case rebar_state:get(GlobalConfigThrowAway1, plugins, []) of [] -> - GlobalConfigThrowAway; + GlobalConfigThrowAway1; GlobalPluginsToInstall -> rebar_plugins:handle_plugins(global, GlobalPluginsToInstall, - GlobalConfigThrowAway) + GlobalConfigThrowAway1) end, GlobalPlugins = rebar_state:providers(GlobalState), GlobalConfig2 = rebar_state:set(GlobalConfig, plugins, []), GlobalConfig3 = rebar_state:set(GlobalConfig2, {plugins, global}, - rebar_state:get(GlobalConfigThrowAway, plugins, [])), + rebar_state:get(GlobalConfigThrowAway1, plugins, [])), rebar_state:providers(rebar_state:new(GlobalConfig3, Config), GlobalPlugins). -spec test_state(rebar_state:t()) -> [{'extra_src_dirs',[string()]} | {'erl_opts',[any()]}]. From 374f23c5807d2e89de6be148761b0201cbfad36a Mon Sep 17 00:00:00 2001 From: getong <3949379+getong@users.noreply.github.com> Date: Sat, 10 Nov 2018 11:18:54 +0800 Subject: [PATCH 9/9] fix install output result --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8075535f..e312ca64 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,7 @@ The rebar3 escript can also extract itself with a run script under the user's ho $ ./rebar3 local install ===> Extracting rebar3 libs to ~/.cache/rebar3/lib... ===> Writing rebar3 run script ~/.cache/rebar3/bin/rebar3... -===> Add to $PATH for use: export PATH=$PATH:~/.cache/rebar3/bin +===> Add to $PATH for use: export PATH=~/.cache/rebar3/bin:$PATH ``` To keep it up to date after you've installed rebar3 this way you can use `rebar3 local upgrade` which