Proper Unicode handling in port process output

The problem is that the shell util handlers receive a byte stream, which
we handle as a list coming from a port. However the byte stream is utf-8
encoded in most systems, which breaks encoding expectations (binaries
only) for Erlang.

This patch turns the list of characters to a binary for all shell calls,
before re-encoding it properly as unicode lists to keep the format compatible.

Fixes #2205

src/rebar_utils.erl

@@ -189,7 +189,7 @@ sh(Command0, Options0) ->
 
     Command = lists:flatten(patch_on_windows(Command0, proplists:get_value(env, Options0, []))),
     PortSettings = proplists:get_all_values(port_settings, Options) ++
-        [exit_status, {line, 16384}, use_stdio, stderr_to_stdout, hide, eof],
+        [exit_status, {line, 16384}, use_stdio, stderr_to_stdout, hide, eof, binary],
     ?DEBUG("Port Cmd: ~ts\nPort Opts: ~p\n", [Command, PortSettings]),
     Port = open_port({spawn, Command}, PortSettings),
 
@@ -676,9 +676,9 @@ debug_and_abort(Command, {Rc, Output}) ->
 sh_loop(Port, Fun, Acc) ->
     receive
         {Port, {data, {eol, Line}}} ->
-            sh_loop(Port, Fun, Fun(Line ++ "\n", Acc));
+            sh_loop(Port, Fun, Fun(unicode:characters_to_list(Line) ++ "\n", Acc));
         {Port, {data, {noeol, Line}}} ->
-            sh_loop(Port, Fun, Fun(Line, Acc));
+            sh_loop(Port, Fun, Fun(unicode:characters_to_list(Line), Acc));
         {Port, eof} ->
             Data = lists:flatten(lists:reverse(Acc)),
             receive