Merge pull request #1061 from ferd/future-proof-lockfiles

Make lock files future-proof

src/rebar_app_info.erl

@@ -165,13 +165,13 @@ update_opts(AppInfo, Opts, Config) ->
 
 deps_from_config(Dir, Config) ->
     case rebar_config:consult_lock_file(filename:join(Dir, ?LOCK_FILE)) of
-        [D] ->
+        [] ->
+            [{{deps, default}, proplists:get_value(deps, Config, [])}];
+        D ->
             %% We want the top level deps only from the lock file.
             %% This ensures deterministic overrides for configs.
             Deps = [X || X <- D, element(3, X) =:= 0],
-            [{{locks, default}, D}, {{deps, default}, Deps}];
-        _ ->
-            [{{deps, default}, proplists:get_value(deps, Config, [])}]
+            [{{locks, default}, D}, {{deps, default}, Deps}]
     end.
 
 %% @doc discover a complete version of the app info with all fields set.

src/rebar_config.erl

@@ -30,6 +30,7 @@
         ,consult_app_file/1
         ,consult_file/1
         ,consult_lock_file/1
+        ,write_lock_file/2
         ,verify_config_format/1
         ,format_error/1
 
@@ -50,7 +51,40 @@ consult_app_file(File) ->
     consult_file_(File).
 
 consult_lock_file(File) ->
-    consult_file_(File).
+    Terms = consult_file_(File),
+    case Terms of
+        [] ->
+            [];
+        [Locks] when is_list(Locks) -> % beta lock file
+            Locks;
+        [{Vsn, Locks}|Attrs] when is_list(Locks) -> % versioned lock file
+            %% Make sure the warning above is to be shown whenever a version
+            %% newer than the current one is being used, as we can't parse
+            %% all the contents of the lock file properly.
+            ?WARN("Rebar3 detected a lock file from a newer version. "
+                  "It will be loaded in compatibility mode, but important "
+                  "information may be missing or lost. It is recommended to "
+                  "upgrade Rebar3.", []),
+            read_attrs(Vsn, Locks, Attrs)
+    end.
+
+write_lock_file(LockFile, Locks) ->
+    NewLocks = write_attrs(Locks),
+    %% Write locks in the beta format, at least until it's been long
+    %% enough we can start modifying the lock format.
+    file:write_file(LockFile, io_lib:format("~p.~n", [NewLocks])).
+
+read_attrs(_Vsn, Locks, _Attrs) ->
+    %% Beta copy does not know how to expand attributes, but
+    %% is ready to support it.
+    Locks.
+
+write_attrs(Locks) ->
+    %% No attribute known that needs to be taken out of the structure,
+    %% just return terms as is.
+    Locks.
+
+
 
 consult_file(File) ->
     Terms = consult_file_(File),
@@ -87,7 +121,7 @@ verify_config_format([Term | _]) ->
 merge_locks(Config, []) ->
     Config;
 %% lockfile with entries
-merge_locks(Config, [Locks]) ->
+merge_locks(Config, Locks) ->
     ConfigDeps = proplists:get_value(deps, Config, []),
     %% We want the top level deps only from the lock file.
     %% This ensures deterministic overrides for configs.

src/rebar_prv_lock.erl

@@ -35,8 +35,7 @@ do(State) ->
             OldLocks = rebar_state:get(State, {locks, default}, []),
             Locks = lists:keysort(1, build_locks(State)),
             Dir = rebar_state:dir(State),
-            file:write_file(filename:join(Dir, ?LOCK_FILE),
-                            io_lib:format("~p.~n", [Locks])),
+            rebar_config:write_lock_file(filename:join(Dir, ?LOCK_FILE), Locks),
             State1 = rebar_state:set(State, {locks, default}, Locks),
 
             OldLockNames = [element(1,L) || L <- OldLocks],

src/rebar_prv_unlock.erl

@@ -46,15 +46,14 @@ do(State) ->
             {ok, State};
         {error, Reason} ->
             ?PRV_ERROR({file,Reason});
-        {ok, [Locks]} ->
+        {ok, _} ->
+            Locks = rebar_config:consult_lock_file(LockFile),
             case handle_unlocks(State, Locks, LockFile) of
                 ok ->
                     {ok, State};
                 {error, Reason} ->
                     ?PRV_ERROR({file,Reason})
-            end;
-        {ok, _Other} ->
-            ?PRV_ERROR(unknown_lock_format)
+            end
     end.
 
 -spec format_error(any()) -> iolist().
@@ -74,7 +73,7 @@ handle_unlocks(State, Locks, LockFile) ->
         _ when Names =:= [] -> % implicitly all locks
             file:delete(LockFile);
         NewLocks ->
-            file:write_file(LockFile, io_lib:format("~p.~n", [NewLocks]))
+            rebar_config:write_lock_file(LockFile, NewLocks)
     end.
 
 parse_names(Bin) ->

src/rebar_state.erl

@@ -115,13 +115,13 @@ new(ParentState, Config, Deps, Dir) ->
 
 deps_from_config(Dir, Config) ->
     case rebar_config:consult_lock_file(filename:join(Dir, ?LOCK_FILE)) of
-        [D] ->
+        [] ->
+            [{{deps, default}, proplists:get_value(deps, Config, [])}];
+        D ->
             %% We want the top level deps only from the lock file.
             %% This ensures deterministic overrides for configs.
             Deps = [X || X <- D, element(3, X) =:= 0],
-            [{{locks, default}, D}, {{deps, default}, Deps}];
-        _ ->
-            [{{deps, default}, proplists:get_value(deps, Config, [])}]
+            [{{locks, default}, D}, {{deps, default}, Deps}]
     end.
 
 base_state() ->

test/rebar_lock_SUITE.erl

@@ -0,0 +1,46 @@
+%%% Most locking tests are implicit in other test suites handling
+%%% dependencies.
+%%% This suite is to test the compatibility layers between various
+%%% versions of lockfiles.
+-module(rebar_lock_SUITE).
+-compile(export_all).
+-include_lib("common_test/include/ct.hrl").
+-include_lib("eunit/include/eunit.hrl").
+
+all() -> [current_version, future_versions_no_attrs, future_versions_attrs].
+
+current_version(Config) ->
+    %% Current version just dumps the locks as is on disk.
+    LockFile = filename:join(?config(priv_dir, Config), "current_version"),
+    Locks = [{<<"app1">>, {git,"some_url", {ref,"some_ref"}}, 2},
+             {<<"app2">>, {git,"some_url", {ref,"some_ref"}}, 0},
+             {<<"app3">>, {hg,"some_url", {ref,"some_ref"}}, 1},
+             {<<"pkg1">>,{pkg,<<"name">>,<<"0.1.6">>},3}],
+    file:write_file(LockFile, io_lib:format("~p.~n", [Locks])),
+    ?assertEqual(Locks, rebar_config:consult_lock_file(LockFile)).
+
+future_versions_no_attrs(Config) ->
+    %% Future versions will keep the same core attribute in there, but
+    %% will do so under a new format bundled with a version and potentially
+    %% some trailing attributes
+    LockFile = filename:join(?config(priv_dir, Config), "future_versions"),
+    Locks = [{<<"app1">>, {git,"some_url", {ref,"some_ref"}}, 2},
+             {<<"app2">>, {git,"some_url", {ref,"some_ref"}}, 0},
+             {<<"app3">>, {hg,"some_url", {ref,"some_ref"}}, 1},
+             {<<"pkg1">>,{pkg,<<"name">>,<<"0.1.6">>},3}],
+    LockData = {"3.5.2", Locks},
+    file:write_file(LockFile, io_lib:format("~p.~n", [LockData])),
+    ?assertEqual(Locks, rebar_config:consult_lock_file(LockFile)).
+
+future_versions_attrs(Config) ->
+    %% Future versions will keep the same core attribute in there, but
+    %% will do so under a new format bundled with a version and potentially
+    %% some trailing attributes
+    LockFile = filename:join(?config(priv_dir, Config), "future_versions"),
+    Locks = [{<<"app1">>, {git,"some_url", {ref,"some_ref"}}, 2},
+             {<<"app2">>, {git,"some_url", {ref,"some_ref"}}, 0},
+             {<<"app3">>, {hg,"some_url", {ref,"some_ref"}}, 1},
+             {<<"pkg1">>,{pkg,<<"name">>,<<"0.1.6">>},3}],
+    LockData = {"3.5.2", Locks},
+    file:write_file(LockFile, io_lib:format("~p.~na.~n{b,c}.~n[d,e,f].~n", [LockData])),
+    ?assertEqual(Locks, rebar_config:consult_lock_file(LockFile)).