From cdc35ea7accc929161654db9be2e8687a565be83 Mon Sep 17 00:00:00 2001 From: Michael Klishin Date: Fri, 18 Dec 2020 07:31:26 +0300 Subject: [PATCH 1/3] Erlang/OTP 25 compatibility for ./bootstrap Replaces (deprecated in OTP 25) http_uri calls with uri_string. --- bootstrap | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bootstrap b/bootstrap index cd41321f..fdd76d33 100755 --- a/bootstrap +++ b/bootstrap @@ -156,7 +156,7 @@ set_httpc_options(_, []) -> ok; set_httpc_options(Scheme, Proxy) -> - {ok, {_, UserInfo, Host, Port, _, _}} = http_uri:parse(Proxy), + #{userinfo := UserInfo, host := Host, port := Port} = uri_string:parse(Proxy), httpc:set_options([{Scheme, {{Host, Port}, []}}], rebar), proxy_ipfamily(Host, inet:gethostbyname(Host)), set_proxy_auth(UserInfo). @@ -721,7 +721,7 @@ set_proxy_auth(UserInfo) -> [Username, Password] = re:split(UserInfo, ":", [{return, list}, {parts,2}, unicode]), %% password may contain url encoded characters, need to decode them first - put(proxy_auth, [{proxy_auth, {Username, http_uri:decode(Password)}}]). + put(proxy_auth, [{proxy_auth, {Username, uri_string:percent_decode(Password)}}]). get_proxy_auth() -> case get(proxy_auth) of From 1f9b77dae03e2dddc8d7517f1e8a10d6635b0713 Mon Sep 17 00:00:00 2001 From: Michael Klishin Date: Thu, 24 Dec 2020 08:54:39 +0300 Subject: [PATCH 2/3] rabar_uri: backport percent_decode/1 from OTP 23.2 and use it to avoid a warning in ./bootstrap. --- bootstrap | 4 +-- src/rebar_uri.erl | 83 ++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 84 insertions(+), 3 deletions(-) diff --git a/bootstrap b/bootstrap index fdd76d33..8c64cbe4 100755 --- a/bootstrap +++ b/bootstrap @@ -156,7 +156,7 @@ set_httpc_options(_, []) -> ok; set_httpc_options(Scheme, Proxy) -> - #{userinfo := UserInfo, host := Host, port := Port} = uri_string:parse(Proxy), + #{userinfo := UserInfo, host := Host, port := Port} = rebar_uri:parse(Proxy), httpc:set_options([{Scheme, {{Host, Port}, []}}], rebar), proxy_ipfamily(Host, inet:gethostbyname(Host)), set_proxy_auth(UserInfo). @@ -721,7 +721,7 @@ set_proxy_auth(UserInfo) -> [Username, Password] = re:split(UserInfo, ":", [{return, list}, {parts,2}, unicode]), %% password may contain url encoded characters, need to decode them first - put(proxy_auth, [{proxy_auth, {Username, uri_string:percent_decode(Password)}}]). + put(proxy_auth, [{proxy_auth, {Username, rebar_uri:percent_decode(Password)}}]). get_proxy_auth() -> case get(proxy_auth) of diff --git a/src/rebar_uri.erl b/src/rebar_uri.erl index 428e4870..a928ef16 100644 --- a/src/rebar_uri.erl +++ b/src/rebar_uri.erl @@ -3,9 +3,22 @@ -export([ parse/1, parse/2, scheme_defaults/0, - append_path/2 + append_path/2, percent_decode/1 ]). +-type error() :: {error, atom(), term()}. + +-define(DEC2HEX(X), + if ((X) >= 0) andalso ((X) =< 9) -> (X) + $0; + ((X) >= 10) andalso ((X) =< 15) -> (X) + $A - 10 + end). + +-define(HEX2DEC(X), + if ((X) >= $0) andalso ((X) =< $9) -> (X) - $0; + ((X) >= $A) andalso ((X) =< $F) -> (X) - $A + 10; + ((X) >= $a) andalso ((X) =< $f) -> (X) - $a + 10 + end). + -ifdef(OTP_RELEASE). -spec parse(URIString) -> URIMap when URIString :: uri_string:uri_string(), @@ -87,6 +100,33 @@ append_path(Url, ExtraPath) -> end. -endif. +%% Taken from OTP 23.2 +-spec percent_decode(URI) -> Result when + URI :: uri_string:uri_string() | uri_string:uri_map(), + Result :: uri_string:uri_string() | + uri_string:uri_map() | + {error, {invalid, {atom(), {term(), term()}}}}. +percent_decode(URIMap) when is_map(URIMap)-> + Fun = fun (K,V) when K =:= userinfo; K =:= host; K =:= path; + K =:= query; K =:= fragment -> + case raw_decode(V) of + {error, Reason, Input} -> + throw({error, {invalid, {K, {Reason, Input}}}}); + Else -> + Else + end; + %% Handle port and scheme + (_,V) -> + V + end, + try maps:map(Fun, URIMap) + catch throw:Return -> + Return + end; +percent_decode(URI) when is_list(URI) orelse + is_binary(URI) -> + raw_decode(URI). + %% OTP 21+ -ifdef(OTP_RELEASE). scheme_defaults() -> @@ -129,3 +169,44 @@ maybe_port(Url, Host, Port, PathQ) -> false -> Host ++ PathQ % port was implicit end. -endif. + +-spec raw_decode(list()|binary()) -> list() | binary() | error(). +raw_decode(Cs) -> + raw_decode(Cs, <<>>). + +raw_decode(L, Acc) when is_list(L) -> + try + B0 = unicode:characters_to_binary(L), + B1 = raw_decode(B0, Acc), + unicode:characters_to_list(B1) + catch + throw:{error, Atom, RestData} -> + {error, Atom, RestData} + end; +raw_decode(<<$%,C0,C1,Cs/binary>>, Acc) -> + case is_hex_digit(C0) andalso is_hex_digit(C1) of + true -> + B = ?HEX2DEC(C0)*16+?HEX2DEC(C1), + raw_decode(Cs, <>); + false -> + throw({error,invalid_percent_encoding,<<$%,C0,C1>>}) + end; +raw_decode(<>, Acc) -> + raw_decode(Cs, <>); +raw_decode(<<>>, Acc) -> + check_utf8(Acc). + +-spec is_hex_digit(char()) -> boolean(). +is_hex_digit(C) + when $0 =< C, C =< $9;$a =< C, C =< $f;$A =< C, C =< $F -> true; +is_hex_digit(_) -> false. + +%% Returns Cs if it is utf8 encoded. +check_utf8(Cs) -> + case unicode:characters_to_list(Cs) of + {incomplete,_,_} -> + throw({error,invalid_utf8,Cs}); + {error,_,_} -> + throw({error,invalid_utf8,Cs}); + _ -> Cs + end. From 45962a1507d69aa2522b0b6201ba96c8a8c20e53 Mon Sep 17 00:00:00 2001 From: Michael Klishin Date: Thu, 24 Dec 2020 10:51:41 +0300 Subject: [PATCH 3/3] rebar_prv_escriptize: ignore hipe dependency on Erlang 24+ as it won't be available. While at it, log errors and more relevant resolution information at debug level. --- src/rebar.app.src | 1 + src/rebar_prv_escriptize.erl | 118 ++++++++++++++++++++++++++++++----- 2 files changed, 102 insertions(+), 17 deletions(-) diff --git a/src/rebar.app.src b/src/rebar.app.src index 303ef27c..b1d3958e 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -8,6 +8,7 @@ {registered, []}, {applications, [kernel, stdlib, + %% note: will be filtered out by rebar_prv_escriptize on Erlang/OTP 24+ hipe, sasl, compiler, diff --git a/src/rebar_prv_escriptize.erl b/src/rebar_prv_escriptize.erl index 38976987..c17ecdc2 100644 --- a/src/rebar_prv_escriptize.erl +++ b/src/rebar_prv_escriptize.erl @@ -169,21 +169,57 @@ format_error(no_main_app) -> get_apps_beams(Apps, AllApps) -> get_apps_beams(Apps, AllApps, []). -get_apps_beams([], _, Acc) -> +-ifdef(OTP_RELEASE). + -if(?OTP_RELEASE>=24). + get_apps_beams(Names, Apps, Acc) -> + get_apps_beams0_on_otp24_plus(Names, Apps, Acc). + -else. + get_apps_beams(Names, Apps, Acc) -> + get_apps_beams0(Names, Apps, Acc). + -endif. +-else. +get_apps_beams(Names, Apps, Acc) -> + get_apps_beams0(Names, Apps, Acc). +-endif. + +-compile({nowarn_unused_function, get_apps_beams0_on_otp24_plus/3}). +get_apps_beams0_on_otp24_plus([], _, Acc) -> Acc; -get_apps_beams([App | Rest], AllApps, Acc) -> +%% hipe is not available on Erlang/OTP 24+ +get_apps_beams0_on_otp24_plus([Name|Names], Apps, Acc) when Name =:= hipe -> + get_apps_beams0_on_otp24_plus(Names, Apps, Acc); +get_apps_beams0_on_otp24_plus([App | Rest], AllApps, Acc) -> case rebar_app_utils:find(rebar_utils:to_binary(App), AllApps) of {ok, App1} -> OutDir = filename:absname(rebar_app_info:ebin_dir(App1)), Beams = get_app_beams(App, OutDir), - get_apps_beams(Rest, AllApps, Beams ++ Acc); + get_apps_beams0_on_otp24_plus(Rest, AllApps, Beams ++ Acc); _-> case code:lib_dir(App, ebin) of {error, bad_name} -> throw(?PRV_ERROR({bad_name, App})); Path -> Beams = get_app_beams(App, Path), - get_apps_beams(Rest, AllApps, Beams ++ Acc) + get_apps_beams0_on_otp24_plus(Rest, AllApps, Beams ++ Acc) + end + end. + +-compile({nowarn_unused_function, get_apps_beams0/3}). +get_apps_beams0([], _, Acc) -> + Acc; +get_apps_beams0([App | Rest], AllApps, Acc) -> + case rebar_app_utils:find(rebar_utils:to_binary(App), AllApps) of + {ok, App1} -> + OutDir = filename:absname(rebar_app_info:ebin_dir(App1)), + Beams = get_app_beams(App, OutDir), + get_apps_beams0(Rest, AllApps, Beams ++ Acc); + _-> + case code:lib_dir(App, ebin) of + {error, bad_name} -> + throw(?PRV_ERROR({bad_name, App})); + Path -> + Beams = get_app_beams(App, Path), + get_apps_beams0(Rest, AllApps, Beams ++ Acc) end end. @@ -246,23 +282,71 @@ get_nonempty(Files) -> find_deps(AppNames, AllApps) -> BinAppNames = [rebar_utils:to_binary(Name) || Name <- AppNames], + ?DEBUG("App dependencies: ~p", [BinAppNames]), [ec_cnv:to_atom(Name) || Name <- find_deps_of_deps(BinAppNames, AllApps, BinAppNames)]. +-ifdef(OTP_RELEASE). + -if(?OTP_RELEASE>=24). + find_deps_of_deps(Names, Apps, Acc) -> + find_deps_of_deps0_on_otp24_plus(Names, Apps, Acc). + -else. + find_deps_of_deps(Names, Apps, Acc) -> + find_deps_of_deps0(Names, Apps, Acc). + -endif. +-else. +find_deps_of_deps(Names, Apps, Acc) -> + find_deps_of_deps0(Names, Apps, Acc). +-endif. + +-compile({nowarn_unused_function, find_deps_of_deps0/3}). %% Should look at the app files to find direct dependencies -find_deps_of_deps([], _, Acc) -> Acc; -find_deps_of_deps([Name|Names], Apps, Acc) -> - ?DIAGNOSTIC("processing ~p", [Name]), - {ok, App} = rebar_app_utils:find(Name, Apps), - DepNames = proplists:get_value(applications, rebar_app_info:app_details(App), []), - BinDepNames = [rebar_utils:to_binary(Dep) || Dep <- DepNames, - %% ignore system libs; shouldn't include them. - DepDir <- [code:lib_dir(Dep)], - DepDir =:= {error, bad_name} orelse % those are all local - not lists:prefix(code:root_dir(), DepDir)] - -- ([Name|Names]++Acc), % avoid already seen deps - ?DIAGNOSTIC("new deps of ~p found to be ~p", [Name, BinDepNames]), - find_deps_of_deps(BinDepNames ++ Names, Apps, BinDepNames ++ Acc). +find_deps_of_deps0([], _, Acc) -> Acc; +find_deps_of_deps0([Name|Names], Apps, Acc) -> + ?DIAGNOSTIC("processing dependencies ~s", [Name]), + ?DEBUG("Looking up dependency ~s, seen: ~p", [Name, Acc]), + case rebar_app_utils:find(Name, Apps) of + {ok, App} -> + DepNames = proplists:get_value(applications, rebar_app_info:app_details(App), []), + ?DEBUG("Discovered app dependencies: ~p", [DepNames]), + + BinDepNames = [rebar_utils:to_binary(Dep) || Dep <- DepNames, + %% ignore system libs; shouldn't include them. + DepDir <- [code:lib_dir(Dep)], + DepDir =:= {error, bad_name} orelse % those are all local + not lists:prefix(code:root_dir(), DepDir)] + -- ([Name|Names]++Acc), % avoid already seen deps + ?DIAGNOSTIC("new deps of ~p found to be ~p", [Name, BinDepNames]), + find_deps_of_deps0(BinDepNames ++ Names, Apps, BinDepNames ++ Acc); + error -> + ?DEBUG("Could not find dependency ~s!", [Name]) + end. + +-compile({nowarn_unused_function, find_deps_of_deps0_on_otp24_plus/3}). +find_deps_of_deps0_on_otp24_plus([], _, Acc) -> Acc; +%% hipe is not available on Erlang/OTP 24+ +find_deps_of_deps0_on_otp24_plus([Name|Names], Apps, Acc) when Name =:= <<"hipe">> -> + find_deps_of_deps0_on_otp24_plus(Names, Apps, Acc); +find_deps_of_deps0_on_otp24_plus([Name|Names], Apps, Acc) -> + ?DIAGNOSTIC("processing dependencies ~s", [Name]), + ?DEBUG("Looking up dependency ~s, seen: ~p", [Name, Acc]), + case rebar_app_utils:find(Name, Apps) of + {ok, App} -> + DepNames = proplists:get_value(applications, rebar_app_info:app_details(App), []), + ?DEBUG("Discovered app dependencies: ~p", [DepNames]), + + BinDepNames = [rebar_utils:to_binary(Dep) || Dep <- DepNames, + %% ignore system libs; shouldn't include them. + DepDir <- [code:lib_dir(Dep)], + DepDir =:= {error, bad_name} orelse % those are all local + not lists:prefix(code:root_dir(), DepDir)] + -- ([Name|Names]++Acc), % avoid already seen deps + ?DIAGNOSTIC("new deps of ~p found to be ~p", [Name, BinDepNames]), + find_deps_of_deps0_on_otp24_plus(BinDepNames ++ Names, Apps, BinDepNames ++ Acc); + error -> + ?DEBUG("Could not find dependency ~s!", [Name]) + end. + def(Rm, State, Key, Default) -> Value0 = rebar_state:get(State, Key, Default),