create rebar_resource behaviour and create git resource

src/rebar_fetch.erl

@@ -7,256 +7,25 @@
 %% -------------------------------------------------------------------
 -module(rebar_fetch).
 
--export([new/4,
-         lock_source/2,
-         download_source/2,
-         update_source1/2,
-         source_engine_avail/1,
-         source_engine_avail/2,
-         has_vcs_dir/2,
-         print_source/1,
-         format_source/2]).
+-export([lock_source/2,
+         download_source/2]).
 
 -include("rebar.hrl").
 
--record(p4_settings, {
-          client=undefined,
-          transport="tcp4:perforce:1666",
-          username,
-          password
-         }).
-
-new(Dir, App, Vsn, Source) ->
-    NewSource = lock_source(Dir, Source),
-    {App, Vsn, NewSource}.
-
-init_p4_settings(Basename) ->
-    #p4_settings{client =
-                     case inet:gethostname() of
-                         {ok,HostName} ->
-                             HostName ++ "-"
-                                 ++ os:getenv("USER") ++ "-"
-                                 ++ Basename
-                                 ++ "-Rebar-automated-download"
-                     end}.
-
-lock_source(AppDir, {git, Url, _}) ->
-    Ref = string:strip(os:cmd("git --git-dir='" ++ AppDir ++ "/.git' rev-parse --verify HEAD"), both, $\n),
-    {git, Url, Ref};
-lock_source(_AppDir, Source) ->
-    Source.
+lock_source(AppDir, Source) ->
+    rebar_git_resource:lock(AppDir, Source).
 
 download_source(AppDir, Source) ->
     TmpDir = ec_file:insecure_mkdtemp(),
     AppDir1 = ec_cnv:to_list(AppDir),
     ec_file:mkdir_p(AppDir1),
-    case download_source_tmp(TmpDir, Source) of
+    case rebar_git_resource:download(TmpDir, Source) of
         {ok, _} ->
             ok = ec_file:copy(TmpDir, filename:absname(AppDir1), [recursive]);
         {tarball, File} ->
             ok = erl_tar:extract(File, [{cwd, TmpDir}
-                                        ,compressed]),
+                                       ,compressed]),
             BaseName = filename:basename(AppDir1),
             [FromDir] = filelib:wildcard(filename:join(TmpDir, BaseName++"-*")),
             ec_file:copy(FromDir, AppDir1, [recursive])
     end.
-
-download_source_tmp(TmpDir, {p4, Url}) ->
-    download_source_tmp(TmpDir, {p4, Url, "#head"});
-download_source_tmp(TmpDir, {p4, Url, Rev}) ->
-    download_source_tmp(TmpDir, {p4, Url, Rev, init_p4_settings(filename:basename(TmpDir))});
-download_source_tmp(TmpDir, {p4, Url, _Rev, Settings}) ->
-    ok = filelib:ensure_dir(TmpDir),
-    rebar_utils:sh_send("p4 client -i",
-                   ?FMT("Client: ~s~n"
-                        ++"Description: generated by Rebar~n"
-                        ++"Root: ~s~n"
-                        ++"View:~n"
-                        ++"  ~s/...  //~s/...~n",
-                        [Settings#p4_settings.client,
-                         TmpDir,
-                         Url,
-                         Settings#p4_settings.client]),
-                       []),
-    rebar_utils:sh(?FMT("p4 -c ~s sync -f", [Settings#p4_settings.client]), []);
-download_source_tmp(TmpDir, {hg, Url, Rev}) ->
-    ok = filelib:ensure_dir(TmpDir),
-    rebar_utils:sh(?FMT("hg clone -U ~s ~s", [Url, filename:basename(TmpDir)]),
-                   [{cd, filename:dirname(TmpDir)}]),
-    rebar_utils:sh(?FMT("hg update ~s", [Rev]), [{cd, TmpDir}]);
-download_source_tmp(TmpDir, {git, Url}) ->
-    download_source_tmp(TmpDir, {git, Url, {branch, "HEAD"}});
-download_source_tmp(TmpDir, {git, Url, ""}) ->
-    download_source_tmp(TmpDir, {git, Url, {branch, "HEAD"}});
-download_source_tmp(TmpDir, {git, Url, {branch, Branch}}) ->
-    ok = filelib:ensure_dir(TmpDir),
-    rebar_utils:sh(?FMT("git clone -n ~s ~s", [Url, filename:basename(TmpDir)]),
-                   [{cd, filename:dirname(TmpDir)}]),
-    rebar_utils:sh(?FMT("git checkout -q origin/~s", [Branch]), [{cd, TmpDir}]);
-download_source_tmp(TmpDir, {git, Url, {tag, Tag}}) ->
-    ok = filelib:ensure_dir(TmpDir),
-    rebar_utils:sh(?FMT("git clone -n ~s ~s", [Url, filename:basename(TmpDir)]),
-                   [{cd, filename:dirname(TmpDir)}]),
-    rebar_utils:sh(?FMT("git checkout -q ~s", [Tag]), [{cd, TmpDir}]);
-download_source_tmp(TmpDir, {git, Url, Rev}) ->
-    ok = filelib:ensure_dir(TmpDir),
-    rebar_utils:sh(?FMT("git clone -n ~s ~s", [Url, filename:basename(TmpDir)]),
-                   [{cd, filename:dirname(TmpDir)}]),
-    rebar_utils:sh(?FMT("git checkout -q ~s", [Rev]), [{cd, TmpDir}]);
-download_source_tmp(TmpDir, {bzr, Url, Rev}) ->
-    ok = filelib:ensure_dir(TmpDir),
-    rebar_utils:sh(?FMT("bzr branch -r ~s ~s ~s",
-                        [Rev, Url, filename:basename(TmpDir)]),
-                   [{cd, filename:dirname(TmpDir)}]);
-download_source_tmp(TmpDir, {svn, Url, Rev}) ->
-    ok = filelib:ensure_dir(TmpDir),
-    rebar_utils:sh(?FMT("svn checkout -r ~s ~s ~s",
-                        [Rev, Url, filename:basename(TmpDir)]),
-                   [{cd, filename:dirname(TmpDir)}]);
-download_source_tmp(TmpDir, {rsync, Url}) ->
-    ok = filelib:ensure_dir(TmpDir),
-    rebar_utils:sh(?FMT("rsync -az --delete ~s/ ~s", [Url, TmpDir]), []);
-download_source_tmp(TmpDir, {fossil, Url}) ->
-    download_source_tmp(TmpDir, {fossil, Url, ""});
-download_source_tmp(TmpDir, {fossil, Url, Version}) ->
-    Repository = filename:join(TmpDir, filename:basename(TmpDir) ++ ".fossil"),
-    ok = filelib:ensure_dir(Repository),
-    ok = file:set_cwd(TmpDir),
-    rebar_utils:sh(?FMT("fossil clone ~s ~s", [Url, Repository]),
-                   [{cd, TmpDir}]),
-    rebar_utils:sh(?FMT("fossil open ~s ~s --nested", [Repository, Version]),
-                   []);
-download_source_tmp(TmpDir, Url) ->
-    TmpFile = filename:join(TmpDir, "package.tar.gz"),
-    {ok, saved_to_file} = httpc:request(get, {binary_to_list(Url), []}, [], [{stream, TmpFile}]),
-    {tarball, TmpFile}.
-
-update_source1(AppDir, Args) when element(1, Args) =:= p4 ->
-    download_source_tmp(AppDir, Args);
-update_source1(AppDir, {git, Url}) ->
-    update_source1(AppDir, {git, Url, {branch, "HEAD"}});
-update_source1(AppDir, {git, Url, ""}) ->
-    update_source1(AppDir, {git, Url, {branch, "HEAD"}});
-update_source1(AppDir, {git, _Url, {branch, Branch}}) ->
-    ShOpts = [{cd, AppDir}],
-    rebar_utils:sh("git fetch origin", ShOpts),
-    rebar_utils:sh(?FMT("git checkout -q ~s", [Branch]), ShOpts),
-    rebar_utils:sh(
-      ?FMT("git pull --ff-only --no-rebase -q origin ~s", [Branch]),ShOpts);
-update_source1(AppDir, {git, _Url, {tag, Tag}}) ->
-    ShOpts = [{cd, AppDir}],
-    rebar_utils:sh("git fetch origin", ShOpts),
-    rebar_utils:sh(?FMT("git checkout -q ~s", [Tag]), ShOpts);
-update_source1(AppDir, {git, _Url, Refspec}) ->
-    ShOpts = [{cd, AppDir}],
-    rebar_utils:sh("git fetch origin", ShOpts),
-    rebar_utils:sh(?FMT("git checkout -q ~s", [Refspec]), ShOpts);
-update_source1(AppDir, {svn, _Url, Rev}) ->
-    rebar_utils:sh(?FMT("svn up -r ~s", [Rev]), [{cd, AppDir}]);
-update_source1(AppDir, {hg, _Url, Rev}) ->
-    rebar_utils:sh(?FMT("hg pull -u -r ~s", [Rev]), [{cd, AppDir}]);
-update_source1(AppDir, {bzr, _Url, Rev}) ->
-    rebar_utils:sh(?FMT("bzr update -r ~s", [Rev]), [{cd, AppDir}]);
-update_source1(AppDir, {rsync, Url}) ->
-    rebar_utils:sh(?FMT("rsync -az --delete ~s/ ~s",[Url,AppDir]),[]);
-update_source1(AppDir, {fossil, Url}) ->
-    update_source1(AppDir, {fossil, Url, ""});
-update_source1(AppDir, {fossil, _Url, Version}) ->
-    ok = file:set_cwd(AppDir),
-    rebar_utils:sh("fossil pull", [{cd, AppDir}]),
-    rebar_utils:sh(?FMT("fossil update ~s", [Version]), []).
-
-%% ===================================================================
-%% Source helper functions
-%% ===================================================================
-
-source_engine_avail(Source) ->
-    Name = element(1, Source),
-    source_engine_avail(Name, Source).
-
-source_engine_avail(Name, Source)
-  when Name == hg; Name == git; Name == svn; Name == bzr; Name == rsync;
-       Name == fossil; Name == p4 ->
-    case vcs_client_vsn(Name) >= required_vcs_client_vsn(Name) of
-        true ->
-            true;
-        false ->
-            ?ABORT("Rebar requires version ~p or higher of ~s to process ~p\n",
-                   [required_vcs_client_vsn(Name), Name, Source])
-    end.
-
-vcs_client_vsn(false, _VsnArg, _VsnRegex) ->
-    false;
-vcs_client_vsn(Path, VsnArg, VsnRegex) ->
-    {ok, Info} = rebar_utils:sh(Path ++ VsnArg, [{env, [{"LANG", "C"}]},
-                                                 {use_stdout, false}]),
-    case re:run(Info, VsnRegex, [{capture, all_but_first, list}]) of
-        {match, Match} ->
-            list_to_tuple([list_to_integer(S) || S <- Match]);
-        _ ->
-            false
-    end.
-
-required_vcs_client_vsn(p4)     -> {2013, 1};
-required_vcs_client_vsn(hg)     -> {1, 1};
-required_vcs_client_vsn(git)    -> {1, 5};
-required_vcs_client_vsn(bzr)    -> {2, 0};
-required_vcs_client_vsn(svn)    -> {1, 6};
-required_vcs_client_vsn(rsync)  -> {2, 0};
-required_vcs_client_vsn(fossil) -> {1, 0}.
-
-vcs_client_vsn(p4) ->
-    vcs_client_vsn(rebar_utils:find_executable("p4"), " -V",
-                   "Rev\\. .*/(\\d+)\\.(\\d)/");
-vcs_client_vsn(hg) ->
-    vcs_client_vsn(rebar_utils:find_executable("hg"), " --version",
-                   "version (\\d+).(\\d+)");
-vcs_client_vsn(git) ->
-    vcs_client_vsn(rebar_utils:find_executable("git"), " --version",
-                   "git version (\\d+).(\\d+)");
-vcs_client_vsn(bzr) ->
-    vcs_client_vsn(rebar_utils:find_executable("bzr"), " --version",
-                   "Bazaar \\(bzr\\) (\\d+).(\\d+)");
-vcs_client_vsn(svn) ->
-    vcs_client_vsn(rebar_utils:find_executable("svn"), " --version",
-                   "svn, version (\\d+).(\\d+)");
-vcs_client_vsn(rsync) ->
-    vcs_client_vsn(rebar_utils:find_executable("rsync"), " --version",
-                   "rsync  version (\\d+).(\\d+)");
-vcs_client_vsn(fossil) ->
-    vcs_client_vsn(rebar_utils:find_executable("fossil"), " version",
-                   "version (\\d+).(\\d+)").
-
-has_vcs_dir(p4, _) ->
-    true;
-has_vcs_dir(git, Dir) ->
-    filelib:is_dir(filename:join(Dir, ".git"));
-has_vcs_dir(hg, Dir) ->
-    filelib:is_dir(filename:join(Dir, ".hg"));
-has_vcs_dir(bzr, Dir) ->
-    filelib:is_dir(filename:join(Dir, ".bzr"));
-has_vcs_dir(svn, Dir) ->
-    filelib:is_dir(filename:join(Dir, ".svn"))
-        orelse filelib:is_dir(filename:join(Dir, "_svn"));
-has_vcs_dir(rsync, _) ->
-    true;
-has_vcs_dir(_, _) ->
-    true.
-
-print_source({App, _, Source}) ->
-    ?CONSOLE("~s~n", [format_source(App, Source)]).
-
-format_source(App, {p4, Url}) ->
-    format_source(App, {p4, Url, "#head"});
-format_source(App, {git, Url}) ->
-    ?FMT("~p BRANCH ~s ~s", [App, "HEAD", Url]);
-format_source(App, {git, Url, ""}) ->
-    ?FMT("~p BRANCH ~s ~s", [App, "HEAD", Url]);
-format_source(App, {git, Url, {branch, Branch}}) ->
-    ?FMT("~p BRANCH ~s ~s", [App, Branch, Url]);
-format_source(App, {git, Url, {tag, Tag}}) ->
-    ?FMT("~p TAG ~s ~s", [App, Tag, Url]);
-format_source(App, {_, Url, Rev}) ->
-    ?FMT("~p REV ~s ~s", [App, Rev, Url]);
-format_source(App, undefined) ->
-    ?FMT("~p", [App]).

src/rebar_git_resource.erl

@@ -0,0 +1,44 @@
+%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*-
+%% ex: ts=4 sw=4 et
+-module(rebar_git_resource).
+
+-behaviour(rebar_resource).
+
+-export([lock/2
+        ,download/2]).
+
+-include("rebar.hrl").
+
+lock(AppDir, {git, Url, _}) ->
+    Ref = string:strip(
+            os:cmd("git --git-dir='" ++ AppDir ++ "/.git' rev-parse --verify HEAD")
+            ,both, $\n),
+    {git, Url, {ref, Ref}}.
+
+download(Dir, {git, Url}) ->
+    ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.~n", []),
+    download(Dir, {git, Url, {branch, "HEAD"}});
+download(Dir, {git, Url, ""}) ->
+    ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.~n", []),
+    download(Dir, {git, Url, {branch, "HEAD"}});
+download(Dir, {git, Url, {branch, Branch}}) ->
+    ok = filelib:ensure_dir(Dir),
+    rebar_utils:sh(?FMT("git clone ~s ~s -b ~s --single-branch",
+                       [Url, filename:basename(Dir), Branch]),
+                   [{cd, filename:dirname(Dir)}]);
+download(Dir, {git, Url, {tag, Tag}}) ->
+    ok = filelib:ensure_dir(Dir),
+    rebar_utils:sh(?FMT("git clone ~s ~s -b ~s --single-branch",
+                        [Url, filename:basename(Dir), Tag]),
+                   [{cd, filename:dirname(Dir)}]);
+download(Dir, {git, Url, {ref, Ref}}) ->
+    ok = filelib:ensure_dir(Dir),
+    rebar_utils:sh(?FMT("git clone -n ~s ~s", [Url, filename:basename(Dir)]),
+                   [{cd, filename:dirname(Dir)}]),
+    rebar_utils:sh(?FMT("git checkout -q ~s", [Ref]), [{cd, Dir}]);
+download(Dir, {git, Url, Rev}) ->
+    ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.~n", []),
+    ok = filelib:ensure_dir(Dir),
+    rebar_utils:sh(?FMT("git clone -n ~s ~s", [Url, filename:basename(Dir)]),
+                   [{cd, filename:dirname(Dir)}]),
+    rebar_utils:sh(?FMT("git checkout -q ~s", [Rev]), [{cd, Dir}]).

src/rebar_prv_install_deps.erl

@@ -116,7 +116,8 @@ handle_deps(State, Deps, Update) ->
                                   SrcDeps),
 
     %% Fetch transitive src deps
-    State2 = update_src_deps(0, State1, Update),
+    {State2, _Seen} = update_src_deps(0, State1, Update, sets:new()),
+
     Solved = case rebar_state:pkg_deps(State2) of
                  [] -> %% No pkg deps
                       [];
@@ -128,7 +129,7 @@ handle_deps(State, Deps, Update) ->
                                  AppInfo <- package_to_app(DepsDir
                                                           ,Packages
                                                           ,Pkg),
-                                 maybe_fetch(AppInfo, Update)]
+                                 maybe_fetch(AppInfo, Update, sets:new())]
              end,
 
     AllDeps = lists:ukeymerge(2
@@ -157,11 +158,13 @@ package_to_app(DepsDir, Packages, Pkg={_, Vsn}) ->
             [rebar_app_info:source(AppInfo2, Link)]
     end.
 
--spec update_src_deps(integer(), rebar_state:t(), boolean()) -> rebar_state:t().
-update_src_deps(Level, State, Update) ->
+-spec update_src_deps(integer(), rebar_state:t(), boolean(), sets:set(binary())) ->
+                             {rebar_state:t(), [binary()]}.
+update_src_deps(Level, State, Update, Seen) ->
     SrcDeps = rebar_state:src_deps(State),
-    case lists:foldl(fun(AppInfo, {SrcDepsAcc, PkgDepsAcc, StateAcc}) ->
-                             case Update of
+    case lists:foldl(fun(AppInfo, {SrcDepsAcc, PkgDepsAcc, StateAcc, SeenAcc}) ->
+                             SeenAcc1 = sets:add_element(rebar_app_info:name(AppInfo), SeenAcc),
+                             {SrcDepsAcc1, PkgDepsAcc1, StateAcc1} = case Update of
                                  {true, UpdateName, UpdateLevel} ->
                                      handle_update(AppInfo
                                                   ,UpdateName
@@ -171,20 +174,21 @@ update_src_deps(Level, State, Update) ->
                                                   ,Level
                                                   ,StateAcc);
                                  _ ->
-                                     maybe_fetch(AppInfo, false),
+                                     maybe_fetch(AppInfo, false, SeenAcc),
                                      handle_dep(AppInfo
                                                ,SrcDepsAcc
                                                ,PkgDepsAcc
                                                ,Level
                                                ,StateAcc)
-                             end
-                     end, {[], rebar_state:pkg_deps(State), State}, SrcDeps) of
-        {[], NewPkgDeps, State1} ->
-            rebar_state:pkg_deps(State1, NewPkgDeps);
-        {NewSrcDeps, NewPkgDeps, State1} ->
+                             end,
+                             {SrcDepsAcc1, PkgDepsAcc1, StateAcc1, SeenAcc1}
+                     end, {[], rebar_state:pkg_deps(State), State, Seen}, SrcDeps) of
+        {[], NewPkgDeps, State1, Seen1} ->
+            {rebar_state:pkg_deps(State1, NewPkgDeps), Seen1};
+        {NewSrcDeps, NewPkgDeps, State1, Seen1} ->
             State2 = rebar_state:pkg_deps(State1, NewPkgDeps),
             State3 = rebar_state:src_deps(State2, NewSrcDeps),
-            update_src_deps(Level+1, State3, Update)
+            update_src_deps(Level+1, State3, Update, Seen1)
     end.
 
 handle_update(AppInfo, UpdateName, UpdateLevel, SrcDeps, PkgDeps, Level, State) ->
@@ -194,7 +198,7 @@ handle_update(AppInfo, UpdateName, UpdateLevel, SrcDeps, PkgDeps, Level, State)
     case UpdateLevel < DepLevel
         orelse Name =:= UpdateName of
         true ->
-            case maybe_fetch(AppInfo, true) of
+            case maybe_fetch(AppInfo, true, []) of
                 true ->
                     handle_dep(AppInfo
                               ,SrcDeps
@@ -228,8 +232,9 @@ handle_dep(DepsDir, AppInfo) ->
     {SrcDeps, PkgDeps} = parse_deps(DepsDir, Deps),
     {AppInfo1, SrcDeps, PkgDeps}.
 
--spec maybe_fetch(rebar_app_info:t(), boolean() | {true, binary(), integer()}) -> boolean().
-maybe_fetch(AppInfo, Update) ->
+-spec maybe_fetch(rebar_app_info:t(), boolean() | {true, binary(), integer()},
+                 sets:set(binary())) -> boolean().
+maybe_fetch(AppInfo, Update, Seen) ->
     AppDir = ec_cnv:to_list(rebar_app_info:dir(AppInfo)),
     Apps = rebar_app_discover:find_apps(["_checkouts"], all),
     case rebar_app_utils:find(rebar_app_info:name(AppInfo), Apps) of
@@ -256,6 +261,8 @@ maybe_fetch(AppInfo, Update) ->
                     rebar_fetch:download_source(AppDir, Source),
                     true;
                 _ ->
+                    io:format("Was ~p seen: ~p~n", [rebar_app_info:name(AppInfo)
+                                                   ,sets:is_element(rebar_app_info:name(AppInfo), Seen)]),
                     false
             end
     end.

src/rebar_resource.erl

@@ -0,0 +1,25 @@
+%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*-
+%% ex: ts=4 sw=4 et
+-module(rebar_resource).
+
+-export([]).
+
+-ifdef(have_callback_support).
+
+%% In the case where R14 or lower is being used to compile the system
+%% we need to export a behaviour info
+-export([behaviour_info/1]).
+
+-spec behaviour_info(atom()) -> [{atom(), arity()}] | undefined.
+behaviour_info(callbacks) ->
+    [{lock, 2},
+     {download, 2}];
+behaviour_info(_) ->
+    undefined.
+
+-else.
+
+-callback lock(string(), tuple()) ->  ok.
+-callback download(string(), tuple()) -> ok.
+
+-endif.