Merge pull request #2036 from ferd/shell-support-cfg-src

Attempt at support sys_config_src in shell

src/rebar_file_utils.erl

@@ -28,6 +28,7 @@
 
 -export([try_consult/1,
          consult_config/2,
+         consult_config_terms/2,
          format_error/1,
          symlink_or_copy/2,
          rm_rf/1,
@@ -66,6 +67,8 @@ try_consult(File) ->
             throw(?PRV_ERROR({bad_term_file, File, Reason}))
     end.
 
+%% @doc Parse a sys.config file and return the configuration terms
+%% for all its potentially nested configs.
 -spec consult_config(rebar_state:t(), string()) -> [[tuple()]].
 consult_config(State, Filename) ->
     Fullpath = filename:join(rebar_dir:root_dir(State), Filename),
@@ -74,6 +77,18 @@ consult_config(State, Filename) ->
         [T] -> T;
         [] -> []
     end,
+    consult_config_terms(State, Config).
+
+%% @doc From a parsed sys.config file, expand all the terms to include
+%% its potential nested configs. It is also possible that no sub-terms
+%% (i.e. the config file does not refer to "some/other/file.config")
+%% that the input term is returned as-is.
+%%
+%% This function is added mostly to help with variable substitution
+%% and evaluation of 'sys.config.src' files, giving a way to handle
+%% expansion that is separate from regular config handling.
+-spec consult_config_terms(rebar_state:t(), [tuple()]) -> [[tuple()]].
+consult_config_terms(State, Config) ->
     JoinedConfig = lists:flatmap(
         fun (SubConfig) when is_list(SubConfig) ->
             case lists:suffix(".config", SubConfig) of

src/rebar_prv_shell.erl

@@ -36,6 +36,7 @@
          format_error/1]).
 
 -include("rebar.hrl").
+-include_lib("providers/include/providers.hrl").
 
 -define(PROVIDER, shell).
 -define(DEPS, [compile]).
@@ -503,7 +504,12 @@ find_config(State) ->
         no_value ->
             no_config;
         Filename when is_list(Filename) ->
-            rebar_file_utils:consult_config(State, Filename)
+            case is_src_config(Filename) of
+                false ->
+                    rebar_file_utils:consult_config(State, Filename);
+                true ->
+                    consult_env_config(State, Filename)
+            end
     end.
 
 -spec first_value([Fun], State) -> no_value | Value when
@@ -540,5 +546,69 @@ find_config_rebar(State) ->
 
 -spec find_config_relx(rebar_state:t()) -> [tuple()] | no_value.
 find_config_relx(State) ->
-    debug_get_value(sys_config, rebar_state:get(State, relx, []), no_value,
-                    "Found config from relx.").
+    %% The order in relx is to load the src version first;
+    %% we do the same.
+    RelxCfg = rebar_state:get(State, relx, []),
+    Src = debug_get_value(sys_config_src, RelxCfg, no_value,
+                          "Found config.src from relx."),
+    case Src of
+        no_value ->
+            debug_get_value(sys_config, RelxCfg, no_value,
+                            "Found config from relx.");
+        _ ->
+            Src
+    end.
+
+-spec is_src_config(file:filename()) -> boolean().
+is_src_config(Filename) ->
+    filename:extension(Filename) =:= ".src".
+
+-spec consult_env_config(rebar_state:t(), file:filename()) -> [[tuple()]].
+consult_env_config(State, Filename) ->
+    RawString = case file:read_file(Filename) of
+        {error, _} -> "[].";
+        {ok, Bin} -> unicode:characters_to_list(Bin)
+    end,
+    ReplacedStr = replace_env_vars(RawString),
+    case rebar_string:consult(unicode:characters_to_list(ReplacedStr)) of
+        {error, Reason} ->
+            throw(?PRV_ERROR({bad_term_file, Filename, Reason}));
+        [Terms] ->
+            rebar_file_utils:consult_config_terms(State, Terms)
+    end.
+
+%% @doc quick and simple variable substitution writeup.
+%% Supports `${varname}' but not `$varname' nor nested
+%% values such as `${my_${varname}}'.
+%% The variable are also defined as only supporting
+%% the form `[a-zA-Z_]+[a-zA-Z0-9_]*' as per the POSIX
+%% standard.
+-spec replace_env_vars(string()) -> unicode:charlist().
+replace_env_vars("") -> "";
+replace_env_vars("${" ++ Str) ->
+    case until_var_end(Str) of
+        {ok, VarName, Rest} ->
+            replace_varname(VarName) ++ replace_env_vars(Rest);
+        error ->
+            "${" ++ replace_env_vars(Str)
+    end;
+replace_env_vars([Char|Str]) ->
+    [Char | replace_env_vars(Str)].
+
+until_var_end(Str) ->
+    case re:run(Str, "([a-zA-Z_]+[a-zA-Z0-9_]*)}", [{capture, [1], list}]) of
+        nomatch ->
+            error;
+        {match, [Name]} ->
+            {ok, Name, drop_varname(Name, Str)}
+    end.
+
+replace_varname(Var) ->
+    %% os:getenv(Var, "") is only available in OTP-18.0
+    case os:getenv(Var) of
+        false -> "";
+        Val -> Val
+    end.
+
+drop_varname("", "}" ++ Str) -> Str;
+drop_varname([_|Var], [_|Str]) -> drop_varname(Var, Str).

src/rebar_string.erl

@@ -1,7 +1,12 @@
 %%% @doc Compatibility module for string functionality
 %%% for pre- and post-unicode support.
+%%%
+%%% Also contains other useful string functionality.
 -module(rebar_string).
+%% Compatibility exports
 -export([join/2, split/2, lexemes/2, trim/3, uppercase/1, lowercase/1, chr/2]).
+%% Util exports
+-export([consult/1]).
 
 -ifdef(unicode_str).
 
@@ -42,3 +47,27 @@ uppercase(Str) -> string:to_upper(Str).
 lowercase(Str) -> string:to_lower(Str).
 chr(Str, Char) -> string:chr(Str, Char).
 -endif.
+
+%% @doc
+%% Given a string or binary, parse it into a list of terms, ala file:consult/1
+-spec consult(unicode:chardata()) -> {error, term()} | [term()].
+consult(Str) ->
+    consult([], unicode:characters_to_list(Str), []).
+
+consult(Cont, Str, Acc) ->
+    case erl_scan:tokens(Cont, Str, 0) of
+        {done, Result, Remaining} ->
+            case Result of
+                {ok, Tokens, _} ->
+                    case erl_parse:parse_term(Tokens) of
+                        {ok, Term} -> consult([], Remaining, [Term | Acc]);
+                        {error, Reason} -> {error, Reason}
+                    end;
+                {eof, _Other} ->
+                    lists:reverse(Acc);
+                {error, Info, _} ->
+                    {error, Info}
+            end;
+        {more, Cont1} ->
+            consult(Cont1, eof, Acc)
+    end.

src/rebar_templater.erl

@@ -59,7 +59,7 @@ list_templates(State) ->
 
 %% Expand a single template's value
 list_template(Files, {Name, Type, File}, State) ->
-    case consult(load_file(Files, Type, File)) of
+    case rebar_string:consult(binary_to_list(load_file(Files, Type, File))) of
         {error, Reason} ->
             {error, {consult, File, Reason}};
         TemplateTerms ->
@@ -158,7 +158,7 @@ drop_var_docs([{K,V}|Rest]) -> [{K,V} | drop_var_docs(Rest)].
 %% Load the template index, resolve all variables, and then execute
 %% the template.
 create({Template, Type, File}, Files, UserVars, Force, State) ->
-    TemplateTerms = consult(load_file(Files, Type, File)),
+    TemplateTerms = rebar_string:consult(binary_to_list(load_file(Files, Type, File))),
     Vars = drop_var_docs(override_vars(UserVars, get_template_vars(TemplateTerms, State))),
     maybe_warn_about_name(Vars),
     TemplateCwd = filename:dirname(File),
@@ -394,31 +394,6 @@ load_file(_Files, file, Name) ->
     {ok, Bin} = file:read_file(Name),
     Bin.
 
-%% Given a string or binary, parse it into a list of terms, ala file:consult/1
-consult(Str) when is_list(Str) ->
-    consult([], Str, []);
-consult(Bin) when is_binary(Bin)->
-    consult([], binary_to_list(Bin), []).
-
-consult(Cont, Str, Acc) ->
-    case erl_scan:tokens(Cont, Str, 0) of
-        {done, Result, Remaining} ->
-            case Result of
-                {ok, Tokens, _} ->
-                    case erl_parse:parse_term(Tokens) of
-                        {ok, Term} -> consult([], Remaining, [Term | Acc]);
-                        {error, Reason} -> {error, Reason}
-                    end;
-                {eof, _Other} ->
-                    lists:reverse(Acc);
-                {error, Info, _} ->
-                    {error, Info}
-            end;
-        {more, Cont1} ->
-            consult(Cont1, eof, Acc)
-    end.
-
-
 write_file(Output, Data, Force) ->
     %% determine if the target file already exists
     FileExists = filelib:is_regular(Output),