Merge branch 'fix_windows_bootstrap' of https://github.com/ingwinlu/rebar3 into ingwinlu-fix_windows_bootstrap

README.md

@@ -13,7 +13,7 @@ configuration work. rebar also provides dependency management, enabling
 application writers to easily re-use common libraries from a variety of
 locations (git, hg, etc).
 
-3.0 Alpha-4
+3.0 Alpha-5
 ====
 
 [DOCUMENTATION](http://www.rebar3.org/v3.0/docs)
@@ -44,6 +44,7 @@ limit scope.
 | new        | Create new rebar project from templates |
 | pkgs       | List available packages |
 | release    | Build release of project |
+| report     | Report on environment and versions for bug reports |
 | shell      | Run shell with project apps in path |
 | tar        | Package release into tarball |
 | update     | Update package index |

bootstrap

@@ -2,6 +2,7 @@
 %% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*-
 %% ex: ft=erlang ts=4 sw=4 et
 
+
 main(_Args) ->
     %% Fetch and build deps required to build rebar3
     BaseDeps = [{providers, []}
@@ -105,12 +106,105 @@ compile_file(File, Opts) ->
 bootstrap_rebar3() ->
     filelib:ensure_dir("_build/default/lib/rebar/ebin/dummy.beam"),
     code:add_path("_build/default/lib/rebar/ebin/"),
-    file:make_symlink(filename:absname("src"), filename:absname("_build/default/lib/rebar/src")),
+    ok = symlink_or_copy(filename:absname("src"),
+                         filename:absname("_build/default/lib/rebar/src")),
     Sources = ["src/rebar_resource.erl" | filelib:wildcard("src/*.erl")],
     [compile_file(X, [{outdir, "_build/default/lib/rebar/ebin/"}
                      ,return | additional_defines()]) || X <- Sources],
     code:add_patha(filename:absname("_build/default/lib/rebar/ebin")).
 
+%%rebar.hrl
+-define(FMT(Str, Args), lists:flatten(io_lib:format(Str, Args))).
+%%/rebar.hrl
+%%rebar_file_utils
+symlink_or_copy(Source, Target) ->
+    Link = case os:type() of
+               {win32, _} ->
+                   Source;
+               _ ->
+                   rebar_dir:make_relative_path(Source, Target)
+           end,
+    case file:make_symlink(Link, Target) of
+        ok ->
+            ok;
+        {error, eexist} ->
+            ok;
+        {error, _} ->
+            cp_r([Source], Target)
+    end.
+
+cp_r([], _Dest) ->
+    ok;
+cp_r(Sources, Dest) ->
+    case os:type() of
+        {unix, _} ->
+            EscSources = [escape_path(Src) || Src <- Sources],
+            SourceStr = string:join(EscSources, " "),
+            os:cmd(?FMT("cp -R ~s \"~s\"", [SourceStr, Dest])),
+            ok;
+        {win32, _} ->
+            lists:foreach(fun(Src) -> ok = cp_r_win32(Src,Dest) end, Sources),
+            ok
+    end.
+
+xcopy_win32(Source,Dest)->
+    R = os:cmd(?FMT("xcopy \"~s\" \"~s\" /q /y /e 2> nul",
+                     [filename:nativename(Source), filename:nativename(Dest)])),
+    case length(R) > 0 of
+        %% when xcopy fails, stdout is empty and and error message is printed
+        %% to stderr (which is redirected to nul)
+        true -> ok;
+        false ->
+            {error, lists:flatten(
+                      io_lib:format("Failed to xcopy from ~s to ~s~n",
+                                    [Source, Dest]))}
+    end.
+
+cp_r_win32({true, SourceDir}, {true, DestDir}) ->
+    %% from directory to directory
+    SourceBase = filename:basename(SourceDir),
+    ok = case file:make_dir(filename:join(DestDir, SourceBase)) of
+             {error, eexist} -> ok;
+             Other -> Other
+         end,
+    ok = xcopy_win32(SourceDir, filename:join(DestDir, SourceBase));
+cp_r_win32({false, Source} = S,{true, DestDir}) ->
+    %% from file to directory
+    cp_r_win32(S, {false, filename:join(DestDir, filename:basename(Source))});
+cp_r_win32({false, Source},{false, Dest}) ->
+    %% from file to file
+    {ok,_} = file:copy(Source, Dest),
+    ok;
+cp_r_win32({true, SourceDir}, {false, DestDir}) ->
+    case filelib:is_regular(DestDir) of
+        true ->
+            %% From directory to file? This shouldn't happen
+            {error, lists:flatten(
+                      io_lib:format("Cannot copy dir (~p) to file (~p)\n",
+                                    [SourceDir, DestDir]))};
+        false ->
+            %% Specifying a target directory that doesn't currently exist.
+            %% So let's attempt to create this directory
+            case filelib:ensure_dir(filename:join(DestDir, "dummy")) of
+                ok ->
+                    ok = xcopy_win32(SourceDir, DestDir);
+                {error, Reason} ->
+                    {error, lists:flatten(
+                              io_lib:format("Unable to create dir ~p: ~p\n",
+                                            [DestDir, Reason]))}
+            end
+    end;
+cp_r_win32(Source,Dest) ->
+    Dst = {filelib:is_dir(Dest), Dest},
+    lists:foreach(fun(Src) ->
+                          ok = cp_r_win32({filelib:is_dir(Src), Src}, Dst)
+                  end, filelib:wildcard(Source)),
+    ok.
+
+escape_path(Str) ->
+    re:replace(Str, "([ ()?])", "\\\\&", [global, {return, list}]).
+%%/rebar_file_utils
+
 setup_env() ->
     %% We don't need or want relx providers loaded yet
     application:load(rebar),

src/rebar.app.src

@@ -3,7 +3,7 @@
 
 {application, rebar,
  [{description, "Rebar: Erlang Build Tool"},
-  {vsn, "3.0.0-alpha-4"},
+  {vsn, "3.0.0-alpha-5"},
   {modules, []},
   {registered, []},
   {applications, [kernel,

src/rebar_app_discover.erl

@@ -159,7 +159,12 @@ find_app(AppDir, Validate) ->
                     case Validate of
                         V when V =:= invalid ; V =:= all ->
                             AppInfo = create_app_info(AppDir, File),
-                            {true, rebar_app_info:app_file_src(AppInfo, File)};
+                            case AppInfo of
+                              {error, Reason} ->
+                                  throw({error, {invalid_app_file, File, Reason}});
+                              _ ->
+                                  {true, rebar_app_info:app_file_src(AppInfo, File)}
+                            end;
                         valid ->
                             false
                     end;
@@ -175,7 +180,7 @@ find_app(AppDir, Validate) ->
 app_dir(AppFile) ->
     filename:join(rebar_utils:droplast(filename:split(filename:dirname(AppFile)))).
 
--spec create_app_info(file:name(), file:name()) -> rebar_app_info:t() | error.
+-spec create_app_info(file:name(), file:name()) -> rebar_app_info:t() | {error, term()}.
 create_app_info(AppDir, AppFile) ->
     case file:consult(AppFile) of
         {ok, [{application, AppName, AppDetails}]} ->
@@ -193,8 +198,8 @@ create_app_info(AppDir, AppFile) ->
                             false
                     end,
             rebar_app_info:dir(rebar_app_info:valid(AppInfo1, Valid), AppDir);
-        _ ->
-            error
+        {error, Reason} ->
+            {error, Reason}
     end.
 
 dedup([]) -> [];

src/rebar_git_resource.erl

@@ -16,7 +16,7 @@ lock(AppDir, {git, Url, _}) ->
 lock(AppDir, {git, Url}) ->
     AbortMsg = io_lib:format("Locking of git dependency failed in ~s", [AppDir]),
     {ok, VsnString} =
-        rebar_utils:sh("git --git-dir=';" ++ AppDir ++ "/.git' rev-parse --verify HEAD",
+        rebar_utils:sh("git --git-dir=\";" ++ AppDir ++ "/.git\" rev-parse --verify HEAD",
                        [{use_stdout, false}, {debug_abort_on_error, AbortMsg}]),
     Ref = string:strip(VsnString, both, $\n),
     {git, Url, {ref, Ref}}.
@@ -125,7 +125,7 @@ collect_default_refcount() ->
     %% timestamp is really important from an ordering perspective.
     AbortMsg1 = "Getting log of git dependency failed in " ++ rebar_dir:get_cwd(),
     {ok, String} =
-        rebar_utils:sh("git log -n 1 --pretty=format:'%h\n' ",
+        rebar_utils:sh("git log -n 1 --pretty=format:\"%h\n\" ",
                        [{use_stdout, false},
                         {debug_abort_on_error, AbortMsg1}]),
     RawRef = string:strip(String, both, $\n),
@@ -135,41 +135,43 @@ collect_default_refcount() ->
         case Tag of
             undefined ->
                 AbortMsg2 = "Getting rev-list of git depedency failed in " ++ rebar_dir:get_cwd(),
-                rebar_utils:sh("git rev-list HEAD | wc -l",
-                               [{use_stdout, false},
-                                {debug_abort_on_error, AbortMsg2}]);
+                {ok, PatchLines} = rebar_utils:sh("git rev-list HEAD",
+                                                  [{use_stdout, false},
+                                                   {debug_abort_on_error, AbortMsg2}]),
+                rebar_utils:line_count(PatchLines);
             _ ->
                 get_patch_count(Tag)
         end,
     {TagVsn, RawRef, RawCount}.
 
-build_vsn_string(Vsn, RawRef, RawCount) ->
+build_vsn_string(Vsn, RawRef, Count) ->
     %% Cleanup the tag and the Ref information. Basically leading 'v's and
     %% whitespace needs to go away.
     RefTag = [".ref", re:replace(RawRef, "\\s", "", [global])],
-    Count = erlang:iolist_to_binary(re:replace(RawCount, "\\s", "", [global])),
 
     %% Create the valid [semver](http://semver.org) version from the tag
     case Count of
-        <<"0">> ->
+        0 ->
             erlang:binary_to_list(erlang:iolist_to_binary(Vsn));
         _ ->
             erlang:binary_to_list(erlang:iolist_to_binary([Vsn, "+build.",
-                                                           Count, RefTag]))
+                                                           integer_to_list(Count), RefTag]))
     end.
 
 get_patch_count(RawRef) ->
     AbortMsg = "Getting rev-list of git dep failed in " ++ rebar_dir:get_cwd(),
     Ref = re:replace(RawRef, "\\s", "", [global]),
-    Cmd = io_lib:format("git rev-list ~s..HEAD | wc -l",
+    Cmd = io_lib:format("git rev-list ~s..HEAD",
                          [Ref]),
-    rebar_utils:sh(Cmd,
-                   [{use_stdout, false},
-                    {debug_abort_on_error, AbortMsg}]).
+    {ok, PatchLines} = rebar_utils:sh(Cmd,
+                                        [{use_stdout, false},
+                                         {debug_abort_on_error, AbortMsg}]),
+    rebar_utils:line_count(PatchLines).
+
 
 parse_tags() ->
     %% Don't abort on error, we want the bad return to be turned into 0.0.0
-    case rebar_utils:sh("git log --oneline --decorate  | fgrep \"tag: \" -1000",
+    case rebar_utils:sh("git log --oneline --no-walk --tags --decorate",
                         [{use_stdout, false}, return_on_error]) of
         {error, _} ->
             {undefined, "0.0.0"};

src/rebar_otp_app.erl

@@ -104,7 +104,7 @@ preprocess(State, AppInfo, AppSrcFile) ->
             A1 = apply_app_vars(AppVars, AppData),
 
             %% AppSrcFile may contain instructions for generating a vsn number
-            Vsn = app_vsn(AppSrcFile),
+            Vsn = app_vsn(AppSrcFile, State),
             A2 = lists:keystore(vsn, 1, A1, {vsn, Vsn}),
 
             %% systools:make_relup/4 fails with {missing_param, registered}
@@ -197,11 +197,12 @@ consult_app_file(Filename) ->
             end
     end.
 
-app_vsn(AppFile) ->
+app_vsn(AppFile, State) ->
     case consult_app_file(AppFile) of
         {ok, [{application, _AppName, AppData}]} ->
             AppDir = filename:dirname(filename:dirname(AppFile)),
-            rebar_utils:vcs_vsn(get_value(vsn, AppData, AppFile), AppDir);
+            Resources = rebar_state:resources(State),
+            rebar_utils:vcs_vsn(get_value(vsn, AppData, AppFile), AppDir, Resources);
         {error, Reason} ->
             ?ABORT("Failed to consult app file ~s: ~p\n", [AppFile, Reason])
     end.

src/rebar_prv_app_discovery.erl

@@ -45,5 +45,13 @@ do(State) ->
 -spec format_error(any()) -> iolist().
 format_error({multiple_app_files, Files}) ->
     io_lib:format("Multiple app files found in one app dir: ~s", [string:join(Files, " and ")]);
+format_error({invalid_app_file, File, Reason}) ->
+    case Reason of 
+        {Line, erl_parse, Description} ->
+            io_lib:format("Invalid app file ~s at line ~b: ~p", 
+                [File, Line, lists:flatten(Description)]);
+        _ ->
+            io_lib:format("Invalid app file ~s: ~p", [File, Reason])
+    end;
 format_error(Reason) ->
     io_lib:format("~p", [Reason]).

src/rebar_prv_deps.erl

@@ -91,20 +91,12 @@ display_dep(_State, {Name, Vsn}) when is_list(Vsn) ->
     ?CONSOLE("~s* (package ~s)", [ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)]);
 display_dep(_State, Name) when is_binary(Name) ->
     ?CONSOLE("~s* (package)", [Name]);
-%% git source
-display_dep(_State, {Name, Source}) when is_tuple(Source), element(1, Source) =:= git ->
-    ?CONSOLE("~s* (git source)", [ec_cnv:to_binary(Name)]);
-display_dep(_State, {Name, _Vsn, Source}) when is_tuple(Source), element(1, Source) =:= git ->
-    ?CONSOLE("~s* (git source)", [ec_cnv:to_binary(Name)]);
-display_dep(_State, {Name, _Vsn, Source, _Opts}) when is_tuple(Source), element(1, Source) =:= git ->
-    ?CONSOLE("~s* (git soutce)", [ec_cnv:to_binary(Name)]);
-%% unknown source
 display_dep(_State, {Name, Source}) when is_tuple(Source) ->
-    ?CONSOLE("~s* (source ~p)", [ec_cnv:to_binary(Name), Source]);
+    ?CONSOLE("~s* (~s source)", [ec_cnv:to_binary(Name), type(Source)]);
 display_dep(_State, {Name, _Vsn, Source}) when is_tuple(Source) ->
-    ?CONSOLE("~s* (source ~p)", [ec_cnv:to_binary(Name), Source]);
+    ?CONSOLE("~s* (~s source)", [ec_cnv:to_binary(Name), type(Source)]);
 display_dep(_State, {Name, _Vsn, Source, _Opts}) when is_tuple(Source) ->
-    ?CONSOLE("~s* (source ~p)", [ec_cnv:to_binary(Name), Source]);
+    ?CONSOLE("~s* (~s source)", [ec_cnv:to_binary(Name), type(Source)]);
 %% Locked
 display_dep(State, {Name, Source={pkg, _, Vsn}, Level}) when is_integer(Level) ->
     DepsDir = rebar_dir:deps_dir(State),
@@ -114,14 +106,6 @@ display_dep(State, {Name, Source={pkg, _, Vsn}, Level}) when is_integer(Level) -
         false -> ""
     end,
     ?CONSOLE("~s~s (locked package ~s)", [Name, NeedsUpdate, Vsn]);
-display_dep(State, {Name, Source, Level}) when is_tuple(Source), is_integer(Level), element(1, Source) =:= git ->
-    DepsDir = rebar_dir:deps_dir(State),
-    AppDir = filename:join([DepsDir, ec_cnv:to_binary(Name)]),
-    NeedsUpdate = case rebar_fetch:needs_update(AppDir, Source, State) of
-        true -> "*";
-        false -> ""
-    end,
-    ?CONSOLE("~s~s (locked git source)", [Name, NeedsUpdate]);
 display_dep(State, {Name, Source, Level}) when is_tuple(Source), is_integer(Level) ->
     DepsDir = rebar_dir:deps_dir(State),
     AppDir = filename:join([DepsDir, ec_cnv:to_binary(Name)]),
@@ -129,4 +113,6 @@ display_dep(State, {Name, Source, Level}) when is_tuple(Source), is_integer(Leve
         true -> "*";
         false -> ""
     end,
-    ?CONSOLE("~s~s (locked ~p)", [Name, NeedsUpdate, Source]).
+    ?CONSOLE("~s~s (locked ~s source)", [Name, NeedsUpdate, type(Source)]).
+
+type(Source) when is_tuple(Source) -> element(1, Source).

src/rebar_prv_shell.erl

@@ -52,7 +52,7 @@ init(State) ->
                                                                {example, "rebar3 shell"},
                                                                {short_desc, "Run shell with project apps and deps in path."},
                                                                {desc, info()},
-                                                               {opts, []}])),
+                                                               {opts, [{config, undefined, "config", string, "Path to the config file to use. Defaults to the sys_config defined for relx, if present."}]}])),
     {ok, State1}.
 
 -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}.
@@ -96,6 +96,8 @@ shell(State) ->
     code:add_pathsa(rebar_state:code_paths(State, all_deps)),
     %% add project app test paths
     ok = add_test_paths(State),
+    %% try to read in sys.config file
+    ok = reread_config(State),
     %% this call never returns (until user quits shell)
     timer:sleep(infinity).
 
@@ -129,3 +131,61 @@ add_test_paths(State) ->
                   end, rebar_state:project_apps(State)),
     _ = code:add_path(filename:join([rebar_dir:base_dir(State), "test"])),
     ok.
+
+reread_config(State) ->
+    case find_config(State) of
+        no_config ->
+            ok;
+        {ok, ConfigList} ->
+            lists:foreach(fun ({Application, Items}) ->
+                                  lists:foreach(fun ({Key, Val}) ->
+                                                        application:set_env(Application, Key, Val)
+                                                end,
+                                                Items)
+                          end,
+                          ConfigList);
+        {error, Error} ->
+            ?ABORT("Error while attempting to read configuration file: ~p", [Error])
+    end.
+
+% First try the --config flag, then try the relx sys_config
+-spec find_config(rebar_state:t()) -> {ok, [tuple()]}|no_config|{error, tuple()}.
+find_config(State) ->
+    case find_config_option(State) of
+        no_config ->
+            find_config_relx(State);
+        Result ->
+            Result
+    end.
+
+-spec find_config_option(rebar_state:t()) -> {ok, [tuple()]}|no_config|{error, tuple()}.
+find_config_option(State) ->
+    {Opts, _} = rebar_state:command_parsed_args(State),
+    case proplists:get_value(config, Opts) of
+        undefined ->
+            no_config;
+        Filename ->
+            consult_config(State, Filename)
+    end.
+
+-spec find_config_relx(rebar_state:t()) -> {ok, [tuple()]}|no_config|{error, tuple()}.
+find_config_relx(State) ->
+    case proplists:get_value(sys_config, rebar_state:get(State, relx, [])) of
+        undefined ->
+            no_config;
+        Filename ->
+            consult_config(State, Filename)
+    end.
+
+-spec consult_config(rebar_state:t(), string()) -> {ok, [tuple()]}|{error, tuple()}.
+consult_config(State, Filename) ->
+    Fullpath = filename:join(rebar_dir:root_dir(State), Filename),
+    ?DEBUG("Loading configuration from ~p", [Fullpath]),
+    case file:consult(Fullpath) of
+        {ok, [Config]} ->
+            {ok, Config};
+        {ok, []} ->
+            {ok, []};
+        {error, Error} ->
+            {error, {Error, Fullpath}}
+    end.

src/rebar_utils.erl

@@ -42,7 +42,7 @@
          erl_to_mod/1,
          beams/1,
          find_executable/1,
-         vcs_vsn/2,
+         vcs_vsn/3,
          deprecated/3,
          deprecated/4,
          erl_opts/1,
@@ -53,7 +53,8 @@
          get_arch/0,
          wordsize/0,
          tup_umerge/2,
-         tup_sort/1]).
+         tup_sort/1,
+         line_count/1]).
 
 %% for internal use only
 -export([otp_release/0]).
@@ -281,6 +282,11 @@ umerge([], Olds, Merged, CmpMerged, Cmp) when CmpMerged == Cmp ->
 umerge([], Olds, Merged, _CmpMerged, Cmp) ->
     lists:reverse(Olds, [Cmp | Merged]).
 
+%% Implements wc -l functionality used to determine patchcount from git output
+line_count(PatchLines) ->
+    Tokenized = string:tokens(PatchLines, "\n"),
+    {ok, length(Tokenized)}.
+
 %% ====================================================================
 %% Internal functions
 %% ====================================================================
@@ -471,8 +477,8 @@ escript_foldl(Fun, Acc, File) ->
             Error
     end.
 
-vcs_vsn(Vcs, Dir) ->
-    case vcs_vsn_cmd(Vcs, Dir) of
+vcs_vsn(Vcs, Dir, Resources) ->
+    case vcs_vsn_cmd(Vcs, Dir, Resources) of
         {plain, VsnString} ->
             VsnString;
         {cmd, CmdString} ->
@@ -484,23 +490,48 @@ vcs_vsn(Vcs, Dir) ->
     end.
 
 %% Temp work around for repos like relx that use "semver"
-vcs_vsn_cmd(VCS, Dir) when VCS =:= semver ; VCS =:= "semver" ->
-    rebar_git_resource:make_vsn(Dir);
-vcs_vsn_cmd(VCS, Dir) when VCS =:= git ; VCS =:= "git" ->
-    rebar_git_resource:make_vsn(Dir);
-vcs_vsn_cmd(VCS, Dir) when VCS =:= pkg ; VCS =:= "pkg" ->
-    rebar_pkg_resource:make_vsn(Dir);
-vcs_vsn_cmd({cmd, _Cmd}=Custom, _) ->
+vcs_vsn_cmd(VCS, Dir, Resources) when VCS =:= semver ; VCS =:= "semver" ->
+    vcs_vsn_cmd(git, Dir, Resources);
+vcs_vsn_cmd({cmd, _Cmd}=Custom, _, _) ->
     Custom;
-vcs_vsn_cmd(Version, _) when is_list(Version) ->
-    {plain, Version};
-vcs_vsn_cmd(_, _) ->
+vcs_vsn_cmd(VCS, Dir, Resources) when is_atom(VCS) ->
+    case find_resource_module(VCS, Resources) of
+        {ok, Module} ->
+            Module:make_vsn(Dir);
+        {error, _} ->
+            unknown
+    end;
+vcs_vsn_cmd(VCS, Dir, Resources) when is_list(VCS) ->
+    try list_to_existing_atom(VCS) of
+        AVCS ->
+            case vcs_vsn_cmd(AVCS, Dir, Resources) of
+                unknown -> {plain, VCS};
+                Other -> Other
+            end
+    catch
+        error:badarg ->
+            {plain, VCS}
+    end;
+vcs_vsn_cmd(_, _, _) ->
     unknown.
 
 vcs_vsn_invoke(Cmd, Dir) ->
     {ok, VsnString} = rebar_utils:sh(Cmd, [{cd, Dir}, {use_stdout, false}]),
     string:strip(VsnString, right, $\n).
 
+find_resource_module(Type, Resources) ->
+    case lists:keyfind(Type, 1, Resources) of
+        false ->
+            case code:which(Type) of
+                non_existing ->
+                    {error, unknown};
+                _ ->
+                    {ok, Type}
+            end;
+        {Type, Module} ->
+            {ok, Module}
+    end.
+
 %%
 %% Filter a list of erl_opts platform_define options such that only
 %% those which match the provided architecture regex are returned.