Make lock files future-proof

Changes to how hex or packages may work in the future will necessarily
bring changes to the format of lock files.

This commit adds an optional framing for future lock files of the form:

    {Version, LockList}.
    <Whatever consultable attributes>

This format is supported such as the LockList is the current lockfile
contents, and will never have more information than it currently does.

Attributes can be whatever and are currently undefined.

Rebar copies will be able to:
- Keep using the core locklist (which avoids breaking the last year or
  so of community libraries using rebar3)
- Warn when it runs an outdated copy in comparison to the lock file
- Automatically rewrite lock files in the format it supports
- Augment or parse files in a version-specific manner.

This changes the usage interface slightly, but is backwards *and*
forwards compatible.

src/rebar_app_info.erl

@@ -165,13 +165,13 @@ update_opts(AppInfo, Opts, Config) ->
 
 deps_from_config(Dir, Config) ->
     case rebar_config:consult_lock_file(filename:join(Dir, ?LOCK_FILE)) of
-        [D] ->
+        [] ->
+            [{{deps, default}, proplists:get_value(deps, Config, [])}];
+        D ->
             %% We want the top level deps only from the lock file.
             %% This ensures deterministic overrides for configs.
             Deps = [X || X <- D, element(3, X) =:= 0],
-            [{{locks, default}, D}, {{deps, default}, Deps}];
-        _ ->
-            [{{deps, default}, proplists:get_value(deps, Config, [])}]
+            [{{locks, default}, D}, {{deps, default}, Deps}]
     end.
 
 %% @doc discover a complete version of the app info with all fields set.

src/rebar_config.erl

@@ -30,6 +30,7 @@
         ,consult_app_file/1
         ,consult_file/1
         ,consult_lock_file/1
+        ,write_lock_file/2
         ,verify_config_format/1
         ,format_error/1
 
@@ -50,7 +51,40 @@ consult_app_file(File) ->
     consult_file_(File).
 
 consult_lock_file(File) ->
-    consult_file_(File).
+    Terms = consult_file_(File),
+    case Terms of
+        [] ->
+            [];
+        [Locks] when is_list(Locks) -> % beta lock file
+            Locks;
+        [{Vsn, Locks}|Attrs] when is_list(Locks) -> % versioned lock file
+            %% Make sure the warning above is to be shown whenever a version
+            %% newer than the current one is being used, as we can't parse
+            %% all the contents of the lock file properly.
+            ?WARN("Rebar3 detected a lock file from a newer version. "
+                  "It will be loaded in compatibility mode, but important "
+                  "information may be missing or lost. It is recommended to "
+                  "upgrade Rebar3.", []),
+            read_attrs(Vsn, Locks, Attrs)
+    end.
+
+write_lock_file(LockFile, Locks) ->
+    NewLocks = write_attrs(Locks),
+    %% Write locks in the beta format, at least until it's been long
+    %% enough we can start modifying the lock format.
+    file:write_file(LockFile, io_lib:format("~p.~n", [NewLocks])).
+
+read_attrs(_Vsn, Locks, _Attrs) ->
+    %% Beta copy does not know how to expand attributes, but
+    %% is ready to support it.
+    Locks.
+
+write_attrs(Locks) ->
+    %% No attribute known that needs to be taken out of the structure,
+    %% just return terms as is.
+    Locks.
+
+
 
 consult_file(File) ->
     Terms = consult_file_(File),
@@ -87,7 +121,7 @@ verify_config_format([Term | _]) ->
 merge_locks(Config, []) ->
     Config;
 %% lockfile with entries
-merge_locks(Config, [Locks]) ->
+merge_locks(Config, Locks) ->
     ConfigDeps = proplists:get_value(deps, Config, []),
     %% We want the top level deps only from the lock file.
     %% This ensures deterministic overrides for configs.

src/rebar_prv_lock.erl

@@ -35,8 +35,7 @@ do(State) ->
             OldLocks = rebar_state:get(State, {locks, default}, []),
             Locks = lists:keysort(1, build_locks(State)),
             Dir = rebar_state:dir(State),
-            file:write_file(filename:join(Dir, ?LOCK_FILE),
-                            io_lib:format("~p.~n", [Locks])),
+            rebar_config:write_lock_file(filename:join(Dir, ?LOCK_FILE), Locks),
             State1 = rebar_state:set(State, {locks, default}, Locks),
 
             OldLockNames = [element(1,L) || L <- OldLocks],

src/rebar_prv_unlock.erl

@@ -46,15 +46,14 @@ do(State) ->
             {ok, State};
         {error, Reason} ->
             ?PRV_ERROR({file,Reason});
-        {ok, [Locks]} ->
+        {ok, _} ->
+            Locks = rebar_config:consult_lock_file(LockFile),
             case handle_unlocks(State, Locks, LockFile) of
                 ok ->
                     {ok, State};
                 {error, Reason} ->
                     ?PRV_ERROR({file,Reason})
-            end;
-        {ok, _Other} ->
-            ?PRV_ERROR(unknown_lock_format)
+            end
     end.
 
 -spec format_error(any()) -> iolist().
@@ -74,7 +73,7 @@ handle_unlocks(State, Locks, LockFile) ->
         _ when Names =:= [] -> % implicitly all locks
             file:delete(LockFile);
         NewLocks ->
-            file:write_file(LockFile, io_lib:format("~p.~n", [NewLocks]))
+            rebar_config:write_lock_file(LockFile, NewLocks)
     end.
 
 parse_names(Bin) ->

src/rebar_state.erl

@@ -115,13 +115,13 @@ new(ParentState, Config, Deps, Dir) ->
 
 deps_from_config(Dir, Config) ->
     case rebar_config:consult_lock_file(filename:join(Dir, ?LOCK_FILE)) of
-        [D] ->
+        [] ->
+            [{{deps, default}, proplists:get_value(deps, Config, [])}];
+        D ->
             %% We want the top level deps only from the lock file.
             %% This ensures deterministic overrides for configs.
             Deps = [X || X <- D, element(3, X) =:= 0],
-            [{{locks, default}, D}, {{deps, default}, Deps}];
-        _ ->
-            [{{deps, default}, proplists:get_value(deps, Config, [])}]
+            [{{locks, default}, D}, {{deps, default}, Deps}]
     end.
 
 base_state() ->

test/rebar_lock_SUITE.erl

@@ -0,0 +1,46 @@
+%%% Most locking tests are implicit in other test suites handling
+%%% dependencies.
+%%% This suite is to test the compatibility layers between various
+%%% versions of lockfiles.
+-module(rebar_lock_SUITE).
+-compile(export_all).
+-include_lib("common_test/include/ct.hrl").
+-include_lib("eunit/include/eunit.hrl").
+
+all() -> [current_version, future_versions_no_attrs, future_versions_attrs].
+
+current_version(Config) ->
+    %% Current version just dumps the locks as is on disk.
+    LockFile = filename:join(?config(priv_dir, Config), "current_version"),
+    Locks = [{<<"app1">>, {git,"some_url", {ref,"some_ref"}}, 2},
+             {<<"app2">>, {git,"some_url", {ref,"some_ref"}}, 0},
+             {<<"app3">>, {hg,"some_url", {ref,"some_ref"}}, 1},
+             {<<"pkg1">>,{pkg,<<"name">>,<<"0.1.6">>},3}],
+    file:write_file(LockFile, io_lib:format("~p.~n", [Locks])),
+    ?assertEqual(Locks, rebar_config:consult_lock_file(LockFile)).
+
+future_versions_no_attrs(Config) ->
+    %% Future versions will keep the same core attribute in there, but
+    %% will do so under a new format bundled with a version and potentially
+    %% some trailing attributes
+    LockFile = filename:join(?config(priv_dir, Config), "future_versions"),
+    Locks = [{<<"app1">>, {git,"some_url", {ref,"some_ref"}}, 2},
+             {<<"app2">>, {git,"some_url", {ref,"some_ref"}}, 0},
+             {<<"app3">>, {hg,"some_url", {ref,"some_ref"}}, 1},
+             {<<"pkg1">>,{pkg,<<"name">>,<<"0.1.6">>},3}],
+    LockData = {"3.5.2", Locks},
+    file:write_file(LockFile, io_lib:format("~p.~n", [LockData])),
+    ?assertEqual(Locks, rebar_config:consult_lock_file(LockFile)).
+
+future_versions_attrs(Config) ->
+    %% Future versions will keep the same core attribute in there, but
+    %% will do so under a new format bundled with a version and potentially
+    %% some trailing attributes
+    LockFile = filename:join(?config(priv_dir, Config), "future_versions"),
+    Locks = [{<<"app1">>, {git,"some_url", {ref,"some_ref"}}, 2},
+             {<<"app2">>, {git,"some_url", {ref,"some_ref"}}, 0},
+             {<<"app3">>, {hg,"some_url", {ref,"some_ref"}}, 1},
+             {<<"pkg1">>,{pkg,<<"name">>,<<"0.1.6">>},3}],
+    LockData = {"3.5.2", Locks},
+    file:write_file(LockFile, io_lib:format("~p.~na.~n{b,c}.~n[d,e,f].~n", [LockData])),
+    ?assertEqual(Locks, rebar_config:consult_lock_file(LockFile)).