Merge pull request #1660 from ferd/otp-21-preparedness

OTP-21 readiness, Full Unicode support

bootstrap

@@ -80,7 +80,7 @@ fetch({pkg, Name, Vsn}, App) ->
         false ->
             CDN = "https://repo.hex.pm/tarballs",
             Package = binary_to_list(<<Name/binary, "-", Vsn/binary, ".tar">>),
-            Url = string:join([CDN, Package], "/"),
+            Url = join([CDN, Package], "/"),
             case request(Url) of
                 {ok, Binary} ->
                     {ok, Contents} = extract(Binary),
@@ -210,7 +210,7 @@ cp_r(Sources, Dest) ->
     case os:type() of
         {unix, _} ->
             EscSources = [escape_path(Src) || Src <- Sources],
-            SourceStr = string:join(EscSources, " "),
+            SourceStr = join(EscSources, " "),
             os:cmd(?FMT("cp -R ~s \"~s\"", [SourceStr, Dest])),
             ok;
         {win32, _} ->
@@ -336,7 +336,11 @@ format_error(AbsSource, Extra, {Mod, Desc}) ->
     io_lib:format("~s: ~s~s~n", [AbsSource, Extra, ErrorDesc]).
 
 additional_defines() ->
-    [{d, D} || {Re, D} <- [{"^[0-9]+", namespaced_types}, {"^R1[4|5]", deprecated_crypto}, {"^((1[8|9])|2)", rand_module}], is_otp_release(Re)].
+    [{d, D} || {Re, D} <- [{"^[0-9]+", namespaced_types},
+                           {"^R1[4|5]", deprecated_crypto},
+                           {"^2", unicode_str},
+                           {"^((1[8|9])|2)", rand_module}],
+               is_otp_release(Re)].
 
 is_otp_release(ArchRegex) ->
     case re:run(otp_release(), ArchRegex, [{capture, none}]) of
@@ -388,9 +392,8 @@ otp_release1(Rel) ->
 set_proxy_auth([]) ->
     ok;
 set_proxy_auth(UserInfo) ->
-    Idx = string:chr(UserInfo, $:), 
-    Username = string:sub_string(UserInfo, 1, Idx-1),
-    Password = string:sub_string(UserInfo, Idx+1),
+    [Username, Password] = re:split(UserInfo, ":",
+                                    [{return, list}, {parts,2}, unicode]),
     %% password may contain url encoded characters, need to decode them first
     put(proxy_auth, [{proxy_auth, {Username, http_uri:decode(Password)}}]).
 
@@ -400,3 +403,12 @@ get_proxy_auth() ->
         ProxyAuth -> ProxyAuth
     end.
 
+%% string:join/2 copy; string:join/2 is getting obsoleted
+%% and replaced by lists:join/2, but lists:join/2 is too new
+%% for version support (only appeared in 19.0) so it cannot be
+%% used. Instead we just adopt join/2 locally and hope it works
+%% for most unicode use cases anyway.
+join([], Sep) when is_list(Sep) ->
+        [];
+join([H|T], Sep) ->
+    H ++ lists:append([Sep ++ X || X <- T]).

rebar.config

@@ -1,16 +1,16 @@
 %% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*-
 %% ex: ts=4 sw=4 ft=erlang et
 
-{deps, [{erlware_commons,     "1.0.0"},
-        {ssl_verify_fun,      "1.1.2"},
+{deps, [{erlware_commons,     "1.0.3"},
+        {ssl_verify_fun,      "1.1.3"},
         {certifi,             "2.0.0"},
-        {providers,           "1.6.0"},
-        {getopt,              "0.8.2"},
+        {providers,           "1.7.0"},
+        {getopt,              "1.0.1"},
         {bbmustache,          "1.3.0"},
-        {relx,                "3.23.1"},
+        {relx,                "3.24.1"},
         {cf,                  "0.2.2"},
-        {cth_readable,        "1.3.0"},
-        {eunit_formatters,    "0.4.0"}]}.
+        {cth_readable,        "1.3.1"},
+        {eunit_formatters,    "0.5.0"}]}.
 
 {post_hooks, [{"(linux|darwin|solaris|freebsd|netbsd|openbsd)",
                escriptize,
@@ -30,6 +30,7 @@
 
 {erl_opts, [{platform_define, "^[0-9]+", namespaced_types},
             {platform_define, "^(19|2)", rand_only},
+            {platform_define, "^2", unicode_str},
             no_debug_info,
             warnings_as_errors]}.
 
@@ -62,6 +63,7 @@
 {overrides, [{override, erlware_commons, [{erl_opts, [{platform_define, "^[0-9]+", namespaced_types},
                                                       {platform_define, "^R1[4|5]", deprecated_crypto},
                                                       {platform_define, "^((1[8|9])|2)", rand_module},
+                                                      {platform_define, "^2", unicode_str},
                                                       no_debug_info,
                                                       warnings_as_errors]},
                                           {deps, []}, {plugins, []}]},

rebar.lock

@@ -2,23 +2,23 @@
 [{<<"bbmustache">>,{pkg,<<"bbmustache">>,<<"1.3.0">>},0},
  {<<"certifi">>,{pkg,<<"certifi">>,<<"2.0.0">>},0},
  {<<"cf">>,{pkg,<<"cf">>,<<"0.2.2">>},0},
- {<<"cth_readable">>,{pkg,<<"cth_readable">>,<<"1.3.0">>},0},
- {<<"erlware_commons">>,{pkg,<<"erlware_commons">>,<<"1.0.0">>},0},
- {<<"eunit_formatters">>,{pkg,<<"eunit_formatters">>,<<"0.4.0">>},0},
- {<<"getopt">>,{pkg,<<"getopt">>,<<"0.8.2">>},0},
- {<<"providers">>,{pkg,<<"providers">>,<<"1.6.0">>},0},
- {<<"relx">>,{pkg,<<"relx">>,<<"3.23.1">>},0},
- {<<"ssl_verify_fun">>,{pkg,<<"ssl_verify_fun">>,<<"1.1.2">>},0}]}.
+ {<<"cth_readable">>,{pkg,<<"cth_readable">>,<<"1.3.1">>},0},
+ {<<"erlware_commons">>,{pkg,<<"erlware_commons">>,<<"1.0.3">>},0},
+ {<<"eunit_formatters">>,{pkg,<<"eunit_formatters">>,<<"0.5.0">>},0},
+ {<<"getopt">>,{pkg,<<"getopt">>,<<"1.0.1">>},0},
+ {<<"providers">>,{pkg,<<"providers">>,<<"1.7.0">>},0},
+ {<<"relx">>,{pkg,<<"relx">>,<<"3.24.1">>},0},
+ {<<"ssl_verify_fun">>,{pkg,<<"ssl_verify_fun">>,<<"1.1.3">>},0}]}.
 [
 {pkg_hash,[
  {<<"bbmustache">>, <<"2010ADAE78830992A4C69680115ECD7D475DD03A72C076BBADDCCBF2D4B32035">>},
  {<<"certifi">>, <<"A0C0E475107135F76B8C1D5BC7EFB33CD3815CB3CF3DEA7AEFDD174DABEAD064">>},
  {<<"cf">>, <<"7F2913FFF90ABCABD0F489896CFEB0B0674F6C8DF6C10B17A83175448029896C">>},
- {<<"cth_readable">>, <<"3F3B4C9CA1C96D5986557A033647A0D7072E25C241AE5EACD894D490EB656706">>},
- {<<"erlware_commons">>, <<"087467DE5833C0BB5B3CCDD387F9E9C1FB816A75B7A709629BF24B5ED3246C51">>},
- {<<"eunit_formatters">>, <<"49B78A45BC06893140DB9FC928307044E0A93794ED90FA181A6F70F2190C1330">>},
- {<<"getopt">>, <<"B17556DB683000BA50370B16C0619DF1337E7AF7ECBF7D64FBF8D1D6BCE3109B">>},
- {<<"providers">>, <<"DB0E2F9043AE60C0155205FCD238D68516331D0E5146155E33D1E79DC452964A">>},
- {<<"relx">>, <<"8AF4433934D9BB664E8282D2E45AC5DEAFF44859DDAABBE50CD7D885581CD24D">>},
- {<<"ssl_verify_fun">>, <<"01289CAD67B280B7F8F7E87117966995FAD19236367386BE2A9D7716E92CE7FF">>}]}
+ {<<"cth_readable">>, <<"53B2C20F823D827A30ABF4F79C93A58DB46934AA038C41792C518BA5FDE9D65B">>},
+ {<<"erlware_commons">>, <<"ABD000FE5893342A405B5F4A2900FF560875B3234E8FE915FDEF172D98EAF250">>},
+ {<<"eunit_formatters">>, <<"6A9133943D36A465D804C1C5B6E6839030434B8879C5600D7DDB5B3BAD4CCB59">>},
+ {<<"getopt">>, <<"C73A9FA687B217F2FF79F68A3B637711BB1936E712B521D8CE466B29CBF7808A">>},
+ {<<"providers">>, <<"BBF730563914328EC2511D205E6477A94831DB7297DE313B3872A2B26C562EAB">>},
+ {<<"relx">>, <<"8211A2C55EA67621C64FDF6A40877D95DA790478B78E3B18496E1C5916046032">>},
+ {<<"ssl_verify_fun">>, <<"6C49665D4326E26CD4A5B7BD54AA442B33DADFB7C5D59A0D0CD0BF5534BBFBD7">>}]}
 ].

src/rebar_dialyzer_format.erl

@@ -427,4 +427,4 @@ separate_args(D, [C | R], Arg, Args) ->
     separate_args(D, R, [C | Arg], Args).
 
 join_args(Args) ->
-    [$(, string:join(Args, ", "), $)].
+    [$(, rebar_string:join(Args, ", "), $)].

src/rebar_dir.erl

@@ -49,7 +49,7 @@ profile_dir(Opts, Profiles) ->
         %%  of profiles to match order passed to `as`
         ["default"|Rest] -> {rebar_opts:get(Opts, base_dir, ?DEFAULT_BASE_DIR), Rest}
     end,
-    ProfilesDir = string:join(ProfilesStrings, "+"),
+    ProfilesDir = rebar_string:join(ProfilesStrings, "+"),
     filename:join(BaseDir, ProfilesDir).
 
 %% @doc returns the directory where dependencies should be placed

src/rebar_file_utils.erl

@@ -190,7 +190,7 @@ cp_r(Sources, Dest) ->
     case os:type() of
         {unix, _} ->
             EscSources = [rebar_utils:escape_chars(Src) || Src <- Sources],
-            SourceStr = string:join(EscSources, " "),
+            SourceStr = rebar_string:join(EscSources, " "),
             {ok, []} = rebar_utils:sh(?FMT("cp -Rp ~ts \"~ts\"",
                                            [SourceStr, rebar_utils:escape_double_quotes(Dest)]),
                                       [{use_stdout, false}, abort_on_error]),

src/rebar_git_resource.erl

@@ -28,7 +28,7 @@ lock(AppDir, {git, Url}) ->
                 rebar_utils:sh("git --git-dir=\"" ++ Dir ++ "/.git\" rev-parse --verify HEAD",
                     [{use_stdout, false}, {debug_abort_on_error, AbortMsg}])
         end,
-    Ref = string:strip(VsnString, both, $\n),
+    Ref = rebar_string:trim(VsnString, both, "
\n"),
     {git, Url, {ref, Ref}}.
 
 %% Return true if either the git url or tag/branch/ref is not the same as the currently
@@ -36,8 +36,8 @@ lock(AppDir, {git, Url}) ->
 needs_update(Dir, {git, Url, {tag, Tag}}) ->
     {ok, Current} = rebar_utils:sh(?FMT("git describe --tags --exact-match", []),
                                    [{cd, Dir}]),
-    Current1 = string:strip(string:strip(Current, both, $\n), both, $\r),
-
+    Current1 = rebar_string:trim(rebar_string:trim(Current, both, "\n"),
+                                 both, "\r"),
     ?DEBUG("Comparing git tag ~ts with ~ts", [Tag, Current1]),
     not ((Current1 =:= Tag) andalso compare_url(Dir, Url));
 needs_update(Dir, {git, Url, {branch, Branch}}) ->
@@ -55,8 +55,8 @@ needs_update(Dir, {git, Url, "master"}) ->
 needs_update(Dir, {git, _, Ref}) ->
     {ok, Current} = rebar_utils:sh(?FMT("git rev-parse --short=7 -q HEAD", []),
                                    [{cd, Dir}]),
-    Current1 = string:strip(string:strip(Current, both, $\n), both, $\r),
-
+    Current1 = rebar_string:trim(rebar_string:trim(Current, both, "\n"),
+                                 both, "\r"),
     Ref2 = case Ref of
                {ref, Ref1} ->
                    Length = length(Current1),
@@ -74,7 +74,8 @@ needs_update(Dir, {git, _, Ref}) ->
 compare_url(Dir, Url) ->
     {ok, CurrentUrl} = rebar_utils:sh(?FMT("git config --get remote.origin.url", []),
                                       [{cd, Dir}]),
-    CurrentUrl1 = string:strip(string:strip(CurrentUrl, both, $\n), both, $\r),
+    CurrentUrl1 = rebar_string:trim(rebar_string:trim(CurrentUrl, both, "\n"),
+                                     both, "\r"),
     {ok, ParsedUrl} = parse_git_url(Url),
     {ok, ParsedCurrentUrl} = parse_git_url(CurrentUrl1),
     ?DEBUG("Comparing git url ~p with ~p", [ParsedUrl, ParsedCurrentUrl]),
@@ -215,7 +216,7 @@ collect_default_refcount(Dir) ->
             ?WARN("Getting log of git dependency failed in ~ts. Falling back to version 0.0.0", [rebar_dir:get_cwd()]),
             {plain, "0.0.0"};
         {ok, String} ->
-            RawRef = string:strip(String, both, $\n),
+            RawRef = rebar_string:trim(String, both, "
\n"),
 
             {Tag, TagVsn} = parse_tags(Dir),
             {ok, RawCount} =
@@ -275,9 +276,9 @@ parse_tags(Dir) ->
                             {undefined, "0.0.0"};
                         %% strip the v prefix if it exists like is done in the above match
                         {ok, [$v | LatestVsn]} ->
-                            {undefined, string:strip(LatestVsn, both, $\n)};
+                            {undefined, rebar_string:trim(LatestVsn, both, "
\n")};
                         {ok, LatestVsn} ->
-                            {undefined, string:strip(LatestVsn, both, $\n)}
+                            {undefined, rebar_string:trim(LatestVsn,both, "
\n")}
                     end
             end
     end.

src/rebar_hg_resource.erl

@@ -96,7 +96,7 @@ make_vsn(Dir) ->
     {ok, VsnString} =
         rebar_utils:sh(Cmd,
                        [{use_stdout, false}, {debug_abort_on_error, AbortMsg}]),
-    RawVsn = string:strip(VsnString, both, $\n),
+    RawVsn = rebar_string:trim(VsnString, both, "
\n"),
 
     Vsn = case RawVsn of
         "null+" ++ Rest -> "0.0.0+" ++ Rest;
@@ -107,8 +107,8 @@ make_vsn(Dir) ->
 %%% Internal functions
 
 compare_url(Dir, Url) ->
-    CurrentUrl = string:strip(os:cmd("hg -R \"" ++ rebar_utils:escape_double_quotes(Dir) ++"\" paths default"), both, $\n),
-    CurrentUrl1 = string:strip(CurrentUrl, both, $\r),
+    CurrentUrl = rebar_string:trim(os:cmd("hg -R \"" ++ rebar_utils:escape_double_quotes(Dir) ++"\" paths default"), both, "
\n"),
+    CurrentUrl1 = rebar_string:trim(CurrentUrl, both, "
\r"),
     parse_hg_url(CurrentUrl1) =:= parse_hg_url(Url).
 
 get_ref(Dir) ->
@@ -116,7 +116,7 @@ get_ref(Dir) ->
     {ok, RefString} =
         rebar_utils:sh("hg -R \"" ++ rebar_utils:escape_double_quotes(Dir) ++ "\" --debug id -i",
                        [{use_stdout, false}, {debug_abort_on_error, AbortMsg}]),
-    string:strip(RefString, both, $\n).
+    rebar_string:trim(RefString, both, "
\n").
 
 get_tag_distance(Dir, Ref) ->
     AbortMsg = io_lib:format("Get tag distance of hg dependency failed in ~ts", [Dir]),
@@ -125,8 +125,8 @@ get_tag_distance(Dir, Ref) ->
                       "log --template \"{latesttag}-{latesttagdistance}\n\" "
                       "--rev " ++ rebar_utils:escape_chars(Ref),
                       [{use_stdout, false}, {debug_abort_on_error, AbortMsg}]),
-    Log = string:strip(LogString,
-                       both, $\n),
+    Log = rebar_string:trim(LogString,
+                            both, "
\n"),
     [Tag, Distance] = re:split(Log, "-([0-9]+)$",
                                [{parts,0}, {return,list}, unicode]),
     {Tag, Distance}.
@@ -137,7 +137,7 @@ get_branch_ref(Dir, Branch) ->
         rebar_utils:sh("hg -R \"" ++ rebar_utils:escape_double_quotes(Dir) ++
                        "\" log --template \"{node}\n\" --rev " ++ rebar_utils:escape_chars(Branch),
                        [{use_stdout, false}, {debug_abort_on_error, AbortMsg}]),
-    string:strip(BranchRefString, both, $\n).
+    rebar_string:strip(BranchRefString, both, "
\n").
 
 
 maybe_warn_local_url(Url) ->
@@ -150,11 +150,11 @@ maybe_warn_local_url(Url) ->
     end.
 
 parse_hg_url("ssh://" ++ HostPath) ->
-    [Host | Path] = string:tokens(HostPath, "/"),
+    [Host | Path] = rebar_string:lexemes(HostPath, "/"),
     {Host, filename:rootname(filename:join(Path), ".hg")};
 parse_hg_url("http://" ++ HostPath) ->
-    [Host | Path] = string:tokens(HostPath, "/"),
+    [Host | Path] = rebar_string:lexemes(HostPath, "/"),
     {Host, filename:rootname(filename:join(Path), ".hg")};
 parse_hg_url("https://" ++ HostPath) ->
-    [Host | Path] = string:tokens(HostPath, "/"),
+    [Host | Path] = rebar_string:lexemes(HostPath, "/"),
     {Host, filename:rootname(filename:join(Path), ".hg")}.

src/rebar_hooks.erl

@@ -140,7 +140,7 @@ create_env(State, Opts) ->
     ].
 
 join_dirs(BaseDir, Dirs) ->
-    string:join([ filename:join(BaseDir, Dir) || Dir <- Dirs ], ":").
+    rebar_string:join([filename:join(BaseDir, Dir) || Dir <- Dirs], ":").
 
 re_version(Path) ->
     case re:run(Path, "^.*-(?<VER>[^/-]*)$", [{capture,[1],list}, unicode]) of

src/rebar_packages.erl

@@ -102,7 +102,7 @@ registry_dir(State) ->
             case rebar_utils:url_append_path(CDN, ?REMOTE_PACKAGE_DIR) of
                 {ok, Parsed} ->
                     {ok, {_, _, Host, _, Path, _}} = http_uri:parse(Parsed),
-                    CDNHostPath = lists:reverse(string:tokens(Host, ".")),
+                    CDNHostPath = lists:reverse(rebar_string:lexemes(Host, ".")),
                     CDNPath = tl(filename:split(Path)),
                     RegistryDir = filename:join([CacheDir, "hex"] ++ CDNHostPath ++ CDNPath),
                     ok = filelib:ensure_dir(filename:join(RegistryDir, "placeholder")),

src/rebar_pkg_resource.erl

@@ -98,7 +98,7 @@ extract(TmpDir, CachePath) ->
 checksums(Pkg={pkg, _Name, _Vsn, Hash}, Files, Contents, Version, Meta, State) ->
     Blob = <<Version/binary, Meta/binary, Contents/binary>>,
     <<X:256/big-unsigned>> = crypto:hash(sha256, Blob),
-    BinChecksum = list_to_binary(string:to_upper(lists:flatten(io_lib:format("~64.16.0b", [X])))),
+    BinChecksum = list_to_binary(rebar_string:uppercase(lists:flatten(io_lib:format("~64.16.0b", [X])))),
     RegistryChecksum = rebar_packages:registry_checksum(Pkg, State),
     {"CHECKSUM", TarChecksum} = lists:keyfind("CHECKSUM", 1, Files),
     {Hash, BinChecksum, RegistryChecksum, TarChecksum}.
@@ -116,7 +116,7 @@ request(Url, ETag) ->
         {ok, {{_Version, 200, _Reason}, Headers, Body}} ->
             ?DEBUG("Successfully downloaded ~ts", [Url]),
             {"etag", ETag1} = lists:keyfind("etag", 1, Headers),
-            {ok, Body, string:strip(ETag1, both, $")};
+            {ok, Body, rebar_string:trim(ETag1, both, [$"])};
         {ok, {{_Version, 304, _Reason}, _Headers, _Body}} ->
             ?DEBUG("Cached copy of ~ts still valid", [Url]),
             {ok, cached};
@@ -132,7 +132,7 @@ etag(Path) ->
     case file:read_file(Path) of
         {ok, Binary} ->
             <<X:128/big-unsigned-integer>> = crypto:hash(md5, Binary),
-            string:to_lower(lists:flatten(io_lib:format("~32.16.0b", [X])));
+            rebar_string:lowercase(lists:flatten(io_lib:format("~32.16.0b", [X])));
         {error, _} ->
             false
     end.
@@ -205,7 +205,7 @@ get_ssl_config() ->
     end.
 
 parse_vsn(Vsn) ->
-    version_pad(string:tokens(Vsn, ".-")).
+    version_pad(rebar_string:lexemes(Vsn, ".-")).
 
 version_pad([Major]) ->
     {list_to_integer(Major), 0, 0};

src/rebar_prv_app_discovery.erl

@@ -49,7 +49,7 @@ do(State) ->
 
 -spec format_error(any()) -> iolist().
 format_error({multiple_app_files, Files}) ->
-    io_lib:format("Multiple app files found in one app dir: ~ts", [string:join(Files, " and ")]);
+    io_lib:format("Multiple app files found in one app dir: ~ts", [rebar_string:join(Files, " and ")]);
 format_error({invalid_app_file, File, Reason}) ->
     case Reason of
         {Line, erl_parse, Description} ->

src/rebar_prv_bare_compile.erl

@@ -42,7 +42,7 @@ do(State) ->
     Paths = proplists:get_value(paths, RawOpts),
     Sep = proplists:get_value(separator, RawOpts, " "),
     [ code:add_pathsa(filelib:wildcard(PathWildcard))
-      || PathWildcard <- string:tokens(Paths, Sep) ],
+      || PathWildcard <- rebar_string:lexemes(Paths, Sep) ],
 
     [AppInfo] = rebar_state:project_apps(State),
     AppInfo1 = rebar_app_info:out_dir(AppInfo, rebar_dir:get_cwd()),

src/rebar_prv_common_test.erl

@@ -173,7 +173,7 @@ transform_opts([Opt|Rest], Acc) ->
     transform_opts(Rest, [Opt|Acc]).
 
 split_string(String) ->
-    string:tokens(String, [$,]).
+    rebar_string:lexemes(String, [$,]).
 
 cfgopts(State) ->
     case rebar_state:get(State, ct_opts, []) of

src/rebar_prv_dialyzer.erl

@@ -565,7 +565,7 @@ collect_nested_dependent_apps(App, Seen) ->
 dialyzer_version() ->
     _ = application:load(dialyzer),
     {ok, Vsn} = application:get_key(dialyzer, vsn),
-    case string:tokens(Vsn, ".") of
+    case rebar_string:lexemes(Vsn, ".") of
         [Major, Minor] ->
             version_tuple(Major, Minor, "0");
         [Major, Minor, Patch | _] ->

src/rebar_prv_eunit.erl

@@ -141,7 +141,8 @@ resolve(Flag, RawOpts) -> resolve(Flag, Flag, RawOpts).
 resolve(Flag, EUnitKey, RawOpts) ->
     case proplists:get_value(Flag, RawOpts) of
         undefined -> [];
-        Args      -> lists:map(fun(Arg) -> normalize(EUnitKey, Arg) end, string:tokens(Args, [$,]))
+        Args      -> lists:map(fun(Arg) -> normalize(EUnitKey, Arg) end,
+                               rebar_string:lexemes(Args, [$,]))
     end.
 
 normalize(Key, Value) when Key == dir; Key == file -> {Key, Value};

src/rebar_prv_local_upgrade.erl

@@ -72,7 +72,7 @@ get_md5(Rebar3Path) ->
     {ok, Rebar3File} = file:read_file(Rebar3Path),
     Digest = crypto:hash(md5, Rebar3File),
     DigestHex = lists:flatten([io_lib:format("~2.16.0B", [X]) || X <- binary_to_list(Digest)]),
-    string:to_lower(DigestHex).
+    rebar_string:lowercase(DigestHex).
 
 maybe_fetch_rebar3(Rebar3Md5) ->
     TmpDir = ec_file:insecure_mkdtemp(),

src/rebar_prv_path.erl

@@ -49,7 +49,7 @@ format_error(Reason) ->
 
 filter_apps(RawOpts, State) ->
     RawApps = proplists:get_all_values(app, RawOpts),
-    Apps = lists:foldl(fun(String, Acc) -> string:tokens(String, ",") ++ Acc end, [], RawApps),
+    Apps = lists:foldl(fun(String, Acc) -> rebar_string:lexemes(String, ",") ++ Acc end, [], RawApps),
     case Apps of
         [] ->
             ProjectDeps = project_deps(State),
@@ -91,7 +91,7 @@ print_paths_if_exist(Paths, State) ->
     {RawOpts, _} = rebar_state:command_parsed_args(State),
     Sep = proplists:get_value(separator, RawOpts, " "),
     RealPaths = lists:filter(fun(P) -> ec_file:is_dir(P) end, Paths),
-    io:format("~ts", [string:join(RealPaths, Sep)]).
+    io:format("~ts", [rebar_string:join(RealPaths, Sep)]).
 
 project_deps(State) ->
     Profiles = rebar_state:current_profiles(State),

src/rebar_prv_shell.erl

@@ -307,7 +307,7 @@ find_apps_option(State) ->
         no_value -> no_value;
         AppsStr ->
             [ list_to_atom(AppStr)
-              || AppStr <- string:tokens(AppsStr, " ,:") ]
+              || AppStr <- rebar_string:lexemes(AppsStr, " ,:") ]
     end.
 
 -spec find_apps_rebar(rebar_state:t()) -> no_value | list().

src/rebar_relx.erl

@@ -27,7 +27,7 @@ do(Module, Command, Provider, State) ->
     LibDirs = rebar_utils:filtermap(fun ec_file:exists/1,
                                    [rebar_dir:checkouts_dir(State), DepsDir | ProjectAppDirs]),
     OutputDir = filename:join(rebar_dir:base_dir(State), ?DEFAULT_RELEASE_DIR),
-    AllOptions = string:join([Command | Options], " "),
+    AllOptions = rebar_string:join([Command | Options], " "),
     Cwd = rebar_state:dir(State),
     Providers = rebar_state:providers(State),
     RebarOpts = rebar_state:opts(State),

src/rebar_string.erl

@@ -0,0 +1,41 @@
+%%% @doc Compatibility module for string functionality
+%%% for pre- and post-unicode support.
+-module(rebar_string).
+-export([join/2, lexemes/2, trim/3, uppercase/1, lowercase/1, chr/2]).
+
+-ifdef(unicode_str).
+
+%% string:join/2 copy; string:join/2 is getting obsoleted
+%% and replaced by lists:join/2, but lists:join/2 is too new
+%% for version support (only appeared in 19.0) so it cannot be
+%% used. Instead we just adopt join/2 locally and hope it works
+%% for most unicode use cases anyway.
+join([], Sep) when is_list(Sep) ->
+        [];
+join([H|T], Sep) ->
+    H ++ lists:append([Sep ++ X || X <- T]).
+
+lexemes(Str, SepList) -> string:lexemes(Str, SepList).
+trim(Str, Direction, Cluster=[_]) -> string:trim(Str, Direction, Cluster).
+uppercase(Str) -> string:uppercase(Str).
+lowercase(Str) -> string:lowercase(Str).
+
+chr(S, C) when is_integer(C) -> chr(S, C, 1).
+chr([C|_Cs], C, I) -> I;
+chr([_|Cs], C, I) -> chr(Cs, C, I+1);
+chr([], _C, _I) -> 0.
+-else.
+
+join(Strings, Separator) -> string:join(Strings, Separator).
+lexemes(Str, SepList) -> string:tokens(Str, SepList).
+trim(Str, Direction, [Char]) ->
+    Dir = case Direction of
+              both -> both;
+              leading -> left;
+              trailing -> right
+          end,
+    string:strip(Str, Dir, Char).
+uppercase(Str) -> string:to_upper(Str).
+lowercase(Str) -> string:to_lower(Str).
+chr(Str, Char) -> string:chr(Str, Char).
+-endif.

src/rebar_templater.erl

@@ -120,7 +120,8 @@ default_author_and_email() ->
         {ok, Name} ->
             case rebar_utils:sh("git config --global user.email", [return_on_error]) of
                 {ok, Email} ->
-                    {string:strip(Name, both, $\n), string:strip(Email, both, $\n)};
+                    {rebar_string:trim(Name, both, "\n"),
+                     rebar_string:trim(Email, both, "\n")};
                 {error, _} ->
                     %% Use neither if one doesn't exist
                     {"Anonymous", "anonymous@example.org"}

src/rebar_utils.erl

@@ -391,7 +391,7 @@ compare({Priority, A}, {Secondary, B}) when not is_tuple(A), is_tuple(B) ->
 
 %% Implements wc -l functionality used to determine patchcount from git output
 line_count(PatchLines) ->
-    Tokenized = string:tokens(PatchLines, "\n"),
+    Tokenized = rebar_string:lexemes(PatchLines, "\n"),
     {ok, length(Tokenized)}.
 
 check_min_otp_version(undefined) ->
@@ -528,7 +528,7 @@ patch_on_windows(Cmd, Env) ->
 %% The end of form `$FOO' is delimited with whitespace or EOL
 -spec expand_env_variable(string(), string(), term()) -> string().
 expand_env_variable(InStr, VarName, RawVarValue) ->
-    case string:chr(InStr, $$) of
+    case rebar_string:chr(InStr, $$) of
         0 ->
             %% No variables to expand
             InStr;
@@ -622,7 +622,7 @@ sh_loop(Port, Fun, Acc) ->
 
 beam_to_mod(Dir, Filename) ->
     [Dir | Rest] = filename:split(Filename),
-    list_to_atom(filename:basename(string:join(Rest, "."), ".beam")).
+    list_to_atom(filename:basename(rebar_string:join(Rest, "."), ".beam")).
 
 beam_to_mod(Filename) ->
     list_to_atom(filename:basename(Filename, ".beam")).
@@ -703,7 +703,7 @@ vcs_vsn_cmd(_, _, _) ->
 
 vcs_vsn_invoke(Cmd, Dir) ->
     {ok, VsnString} = rebar_utils:sh(Cmd, [{cd, Dir}, {use_stdout, false}]),
-    string:strip(VsnString, right, $\n).
+    rebar_string:trim(VsnString, trailing, "
\n").
 
 find_resource_module(Type, Resources) ->
     case lists:keyfind(Type, 1, Resources) of
@@ -896,9 +896,8 @@ list_dir(Dir) ->
 set_proxy_auth([]) ->
     ok;
 set_proxy_auth(UserInfo) ->
-    Idx = string:chr(UserInfo, $:), 
-    Username = string:sub_string(UserInfo, 1, Idx-1),
-    Password = string:sub_string(UserInfo, Idx+1),
+    [Username, Password] = re:split(UserInfo, ":",
+                                    [{return, list}, {parts,2}, unicode]),
     %% password may contain url encoded characters, need to decode them first
     application:set_env(rebar, proxy_auth, [{proxy_auth, {Username, http_uri:decode(Password)}}]).
 

test/rebar_ct_SUITE.erl

@@ -1407,7 +1407,7 @@ testspec_at_root(Config) ->
     CommandProvider = providers:get_provider(ct, Providers, Namespace),
     GetOptSpec = providers:opts(CommandProvider),
 
-    SpecArg1 = string:join([Spec1,Spec2,Spec3],","),
+    SpecArg1 = rebar_string:join([Spec1,Spec2,Spec3],","),
     {ok, GetOptResult1} = getopt:parse(GetOptSpec, ["--spec",SpecArg1]),
     State1 = rebar_state:command_parsed_args(State, GetOptResult1),
     Tests1 = rebar_prv_common_test:prepare_tests(State1),

test/rebar_dialyzer_SUITE.erl

@@ -362,6 +362,6 @@ get_apps_from_beam_files(BeamFiles) ->
     lists:usort(
       [begin
            AppNameVsn = filename:basename(filename:dirname(filename:dirname(File))),
-           [AppName | _] = string:tokens(AppNameVsn ++ "-", "-"),
+           [AppName | _] = rebar_string:lexemes(AppNameVsn ++ "-", "-"),
            ec_cnv:to_atom(AppName)
        end || File <- BeamFiles]).

test/rebar_install_deps_SUITE.erl

@@ -82,12 +82,12 @@ format_expected_mdeps(Deps) ->
     lists:append([
     case Dep of
         {N,V} when hd(N) >= $a, hd(N) =< $z ->
-             UN = string:to_upper(N),
+             UN = rebar_string:uppercase(N),
              [{dep, UN, V}, {lock, pkg, UN, V}];
         {N,V} when hd(N) >= $A, hd(N) =< $Z ->
              [{dep, N, V}, {lock, src, N, V}];
         N when hd(N) >= $a, hd(N) =< $z ->
-             UN = string:to_upper(N),
+             UN = rebar_string:uppercase(N),
              [{dep, UN}, {lock, pkg, UN, "0.0.0"}];
         N when hd(N) >= $A, hd(N) =< $Z ->
              [{dep, N}, {lock, src, N, "0.0.0"}]
@@ -95,9 +95,9 @@ format_expected_mdeps(Deps) ->
 
 format_expected_mixed_warnings(Warnings) ->
     [case W of
-        {N, Vsn} when hd(N) >= $a, hd(N) =< $z -> {pkg, string:to_upper(N), Vsn};
+        {N, Vsn} when hd(N) >= $a, hd(N) =< $z -> {pkg, rebar_string:uppercase(N), Vsn};
         {N, Vsn} when hd(N) >= $A, hd(N) =< $Z -> {git, N, Vsn};
-        N when hd(N) >= $a, hd(N) =< $z -> {pkg, string:to_upper(N), "0.0.0"};
+        N when hd(N) >= $a, hd(N) =< $z -> {pkg, rebar_string:uppercase(N), "0.0.0"};
         N when hd(N) >= $A, hd(N) =< $Z -> {git, N, "0.0.0"}
      end || W <- Warnings].
 

test/rebar_profiles_SUITE.erl

@@ -542,7 +542,7 @@ get_compiled_profile_erl_opts(Profiles, Config) ->
         [default] ->
             ["compile"];
         _ ->
-            ["as", string:join(PStrs, ","), "compile"]
+            ["as", rebar_string:join(PStrs, ","), "compile"]
     end,
     {ok, State} = rebar_test_utils:run_and_check(
         Config, RebarConfig, Command, {ok, [{app, Name}]}),

test/rebar_test_utils.erl

@@ -167,14 +167,14 @@ expand_deps(pkg, [{Name, Vsn, Deps} | Rest]) ->
     [{Dep, expand_deps(pkg, Deps)} | expand_deps(pkg, Rest)];
 expand_deps(mixed, [{Name, Deps} | Rest]) ->
     Dep = if hd(Name) >= $a, hd(Name) =< $z ->
-            {pkg, string:to_upper(Name), "0.0.0", undefined}
+            {pkg, rebar_string:uppercase(Name), "0.0.0", undefined}
            ; hd(Name) >= $A, hd(Name) =< $Z ->
             {Name, ".*", {git, "https://example.org/user/"++Name++".git", "master"}}
     end,
     [{Dep, expand_deps(mixed, Deps)} | expand_deps(mixed, Rest)];
 expand_deps(mixed, [{Name, Vsn, Deps} | Rest]) ->
     Dep = if hd(Name) >= $a, hd(Name) =< $z ->
-            {pkg, string:to_upper(Name), Vsn, undefined}
+            {pkg, rebar_string:uppercase(Name), Vsn, undefined}
            ; hd(Name) >= $A, hd(Name) =< $Z ->
             {Name, Vsn, {git, "https://example.org/user/"++Name++".git", {tag, Vsn}}}
     end,
@@ -477,7 +477,7 @@ package_app(AppDir, DestDir, PkgName) ->
     {ok, Contents} = file:read_file(filename:join(DestDir, "contents.tar.gz")),
     Blob = <<"3who cares", Contents/binary>>,
     <<X:256/big-unsigned>> = crypto:hash(sha256, Blob),
-    BinChecksum = list_to_binary(string:to_upper(lists:flatten(io_lib:format("~64.16.0b", [X])))),
+    BinChecksum = list_to_binary(rebar_string:uppercase(lists:flatten(io_lib:format("~64.16.0b", [X])))),
     ok = file:write_file(filename:join(DestDir, "CHECKSUM"), BinChecksum),
     PkgFiles = ["contents.tar.gz", "VERSION", "metadata.config", "CHECKSUM"],
     Archive = filename:join(DestDir, Name),
@@ -485,5 +485,5 @@ package_app(AppDir, DestDir, PkgName) ->
                         lists:zip(PkgFiles, [filename:join(DestDir,F) || F <- PkgFiles])),
     {ok, BinFull} = file:read_file(Archive),
     <<E:128/big-unsigned-integer>> = crypto:hash(md5, BinFull),
-    Etag = string:to_lower(lists:flatten(io_lib:format("~32.16.0b", [E]))),
+    Etag = rebar_string:lowercase(lists:flatten(io_lib:format("~32.16.0b", [E]))),
     {BinChecksum, Etag}.

test/rebar_unlock_SUITE.erl

@@ -33,7 +33,7 @@ pkgunlock(Config) ->
     rebar_test_utils:run_and_check(Config, [], ["unlock", "cf,certifi"], {ok, []}),
     ?assertEqual(Locks -- ["bbmustache","cf","certifi"], read_locks(Config)),
     ?assertEqual(Hashes -- ["bbmustache","cf","certifi"], read_hashes(Config)),
-    rebar_test_utils:run_and_check(Config, [], ["unlock", string:join(Locks,",")], {ok, []}),
+    rebar_test_utils:run_and_check(Config, [], ["unlock", rebar_string:join(Locks,",")], {ok, []}),
     ?assertEqual({error, enoent}, read_locks(Config)),
     ?assertEqual({error, enoent}, read_hashes(Config)),
     ok.
@@ -48,7 +48,7 @@ unlock(Config) ->
     ?assert(false =/= lists:keyfind(<<"itc">>, 1, rebar_state:get(State, {locks, default}))),
     rebar_test_utils:run_and_check(Config, [], ["unlock", "gproc,itc"], {ok, []}),
     ?assertEqual(Locks -- ["uuid","gproc","itc"], read_locks(Config)),
-    rebar_test_utils:run_and_check(Config, [], ["unlock", string:join(Locks,",")], {ok, []}),
+    rebar_test_utils:run_and_check(Config, [], ["unlock", rebar_string:join(Locks,",")], {ok, []}),
     ?assertEqual({error, enoent}, read_locks(Config)),
     ok.