diff --git a/include/eNet.hrl b/include/eNet.hrl index 81fb7c6..e143061 100644 --- a/include/eNet.hrl +++ b/include/eNet.hrl @@ -1,12 +1,9 @@ --define(nlTcpMgrSup, nlTcpMgrSup). --define(nlSslMgrSup, nlSslMgrSup). --define(nlUdpMgrSup, nlUdpMgrSup). - %% gen_tcp ready maybe to set sock options -%% %% ssl ready and then need do ntSslAcceptor:handshake/3 and maybe to set other options +%% ssl ready and then need do ntSslAcceptor:handshake/3 and maybe to set other options +%% ppt ready and then need do ntPptAcceptor:pptAndHS/5 and maybe to set other options -define(mSockReady, mSockReady). --define(TCP_DEFAULT_OPTIONS, [ +-define(DefTpOpts, [ binary , {packet, 4} , {active, false} @@ -16,15 +13,15 @@ , {send_timeout, 15000} , {keepalive, true} , {exit_on_close, true} + , {back_log, 1024} ]). --define(ACCEPTOR_POOL, 16). - --define(SSL_CLOSE_TIMEOUT, 5000). --define(SSL_HANDSHAKE_TIMEOUT, 15000). --define(PROXY_RECV_TIMEOUT, 5000). +-define(AptCnt, 16). +-define(DefSslHSTet, 15000). +-define(DefProxyPtTet, 5000). +-export_type([listenOpt/0]). -type listenOpt() :: {aptCnt, non_neg_integer()} | {conMod, atom()} | @@ -35,26 +32,10 @@ {proxyPt, boolean()} | {proxyPtTet, timeout()}. --export_type([listenOpt/0]). - %% 令牌桶相关定义 --record(tokenBucket, { +-record(tBucket, { rate :: pos_integer() %% 速率 , tokens :: non_neg_integer() %% 剩余tokens数量 , lastTime :: pos_integer() %% 最后一次更新访问时间单位毫秒 , bucketSize :: pos_integer() %% 桶大小 可以容纳的令牌数量 }). - --type(socket() :: esockd_transport:socket()). --type(mfargs() :: atom() | {atom(), atom()} | {module(), atom(), [term()]}). --type(sock_fun() :: fun((esockd_transport:socket()) -> {ok, esockd_transport:socket()} | {error, term()})). --type(host() :: inet:ip_address() | string()). - - - - - - - - - diff --git a/include/proxyPt.hrl b/include/proxyPt.hrl index 2b8cda8..4080451 100644 --- a/include/proxyPt.hrl +++ b/include/proxyPt.hrl @@ -1,15 +1,6 @@ -ifndef(UT_PROXY_PT_H). -define(UT_PROXY_PT_H, true). -%%-------------------------------------------------------------------- -%% SSL socket wrapper -%%-------------------------------------------------------------------- - --record(ssl_socket, {tcp :: inet:socket() | undefined, %% dtls - ssl :: ssl:sslsocket()}). - --define(IS_SSL(Sock), is_record(Sock, ssl_socket)). - %%-------------------------------------------------------------------- %% Proxy-Protocol Socket Wrapper %%-------------------------------------------------------------------- @@ -30,15 +21,13 @@ | {pp2_netns, binary()} % US-ASCII string | {pp2_ssl, list(pp2_additional_ssl_field())}). --record(proxy_socket, {inet :: inet4 | inet6 | 'unix' | 'unspec', - socket :: inet:socket() | #ssl_socket{}, - src_addr :: inet:ip_address() | undefined, - dst_addr :: inet:ip_address() | undefined, - src_port :: inet:port_number() | undefined, - dst_port :: inet:port_number() | undefined, +-record(proxy_socket, { + inet = inet4 :: inet4 | inet6 | 'unix' | 'unspec', + src_addr = {0,0,0,0} :: inet:ip_address() | undefined, + dst_addr = {0,0,0,0} :: inet:ip_address() | undefined, + src_port = 0 :: inet:port_number() | undefined, + dst_port = 0 :: inet:port_number() | undefined, %% Proxy protocol v2 addtional fields pp2_additional_info = [] :: list(pp2_additional_field())}). --define(IS_PROXY(Sock), is_record(Sock, proxy_socket)). - -endif. diff --git a/src/misc/ntTBucket.erl b/src/misc/ntTBucket.erl index 730e47a..ee2d3ef 100644 --- a/src/misc/ntTBucket.erl +++ b/src/misc/ntTBucket.erl @@ -23,7 +23,7 @@ , check/3 ]). --type(tokenBucket() :: #tokenBucket{}). +-type(tokenBucket() :: #tBucket{}). -type(tbConfig() :: {pos_integer(), pos_integer()}). -spec(new(tbConfig()) -> tokenBucket()). @@ -32,7 +32,7 @@ new({Rate, BucketSize}) -> -spec(new(pos_integer(), pos_integer()) -> tokenBucket()). new(Rate, BucketSize) when is_integer(BucketSize), 0 < Rate andalso Rate =< BucketSize -> - #tokenBucket{ + #tBucket{ rate = Rate , tokens = BucketSize , lastTime = erlang:system_time(milli_seconds) @@ -44,15 +44,15 @@ check(Consume, TokenBucket) -> check(Consume, erlang:system_time(milli_seconds), TokenBucket). -spec(check(pos_integer(), integer(), tokenBucket()) -> {non_neg_integer(), tokenBucket()}). -check(Consume, Now, #tokenBucket{rate = Rate, tokens = Tokens, lastTime = LastTime, bucketSize = BucketSize} = TokenBucket) -> +check(Consume, Now, #tBucket{rate = Rate, tokens = Tokens, lastTime = LastTime, bucketSize = BucketSize} = TokenBucket) -> AvailableToken = erlang:min(BucketSize, Tokens + (Rate * (Now - LastTime)) div 1000), case AvailableToken >= Consume of true -> %% Tokens available - {0, TokenBucket#tokenBucket{tokens = AvailableToken - Consume, lastTime = Now}}; + {0, TokenBucket#tBucket{tokens = AvailableToken - Consume, lastTime = Now}}; false -> %% Tokens not enough %% 计算需要等待的时间 单位毫秒 WaitTime = (Consume - AvailableToken) * 1000 div Rate, - {WaitTime, TokenBucket#tokenBucket{tokens = 0, lastTime = Now}} + {WaitTime, TokenBucket#tBucket{tokens = 0, lastTime = Now}} end. diff --git a/src/proxyPt/esockd_proxy_protocol.erl b/src/proxyPt/esockd_proxy_protocol.erl deleted file mode 100644 index 85af2a1..0000000 --- a/src/proxyPt/esockd_proxy_protocol.erl +++ /dev/null @@ -1,210 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - -%% @doc [Proxy Protocol](https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt) --module(esockd_proxy_protocol). - --include("proxyPt.hrl"). - --export([recv/3]). - --ifdef(TEST). --export([parse_v1/2, parse_v2/4, parse_pp2_tlv/2, parse_pp2_ssl/1]). --endif. - -%% Protocol Command --define(LOCAL, 16#0). --define(PROXY, 16#1). - -%% Address families --define(UNSPEC, 16#0). --define(INET, 16#1). --define(INET6, 16#2). --define(UNIX, 16#3). - --define(STREAM, 16#1). --define(DGRAM, 16#2). - --define(SPACE, 16#20). - --define(TIMEOUT, 5000). - -%% Proxy Protocol Additional Fields --define(PP2_TYPE_ALPN, 16#01). --define(PP2_TYPE_AUTHORITY, 16#02). --define(PP2_TYPE_CRC32C, 16#03). --define(PP2_TYPE_NOOP, 16#04). --define(PP2_TYPE_SSL, 16#20). --define(PP2_SUBTYPE_SSL_VERSION, 16#21). --define(PP2_SUBTYPE_SSL_CN, 16#22). --define(PP2_SUBTYPE_SSL_CIPHER, 16#23). --define(PP2_SUBTYPE_SSL_SIG_ALG, 16#24). --define(PP2_SUBTYPE_SSL_KEY_ALG, 16#25). --define(PP2_TYPE_NETNS, 16#30). - -%% Protocol signature: -%% 16#0D,16#0A,16#00,16#0D,16#0A,16#51,16#55,16#49,16#54,16#0A --define(SIG, "\r\n\0\r\nQUIT\n"). - --spec(recv(module(), inet:socket() | #ssl_socket{}, timeout()) -> - {ok, #proxy_socket{}} | {error, term()}). -recv(Transport, Sock, Timeout) -> - {ok, OriginOpts} = Transport:getopts(Sock, [mode, active, packet]), - ok = Transport:setopts(Sock, [binary, {active, once}, {packet, line}]), - receive - %% V1 TCP - {_, _Sock, <<"PROXY TCP", Proto, ?SPACE, ProxyInfo/binary>>} -> - Transport:setopts(Sock, OriginOpts), - parse_v1(ProxyInfo, #proxy_socket{inet = inet_family(Proto), socket = Sock}); - %% V1 Unknown - {_, _Sock, <<"PROXY UNKNOWN", _ProxyInfo/binary>>} -> - Transport:setopts(Sock, OriginOpts), - {ok, Sock}; - %% V2 TCP - {_, _Sock, <<"\r\n">>} -> - Transport:setopts(Sock, [{active, false}, {packet, raw}]), - {ok, Header} = Transport:recv(Sock, 14, 1000), - <> = Header, - case Transport:recv(Sock, Len, 1000) of - {ok, ProxyInfo} -> - Transport:setopts(Sock, OriginOpts), - parse_v2(Cmd, Trans, ProxyInfo, #proxy_socket{inet = inet_family(AF), socket = Sock}); - {error, Reason} -> - {error, {recv_proxy_info_error, Reason}} - end; - {tcp_error, _Sock, Reason} -> - {error, {recv_proxy_info_error, Reason}}; - {tcp_closed, _Sock} -> - {error, {recv_proxy_info_error, tcp_closed}}; - {_, _Sock, ProxyInfo} -> - {error, {invalid_proxy_info, ProxyInfo}} - after - Timeout -> - {error, proxy_proto_timeout} - end. - -parse_v1(ProxyInfo, ProxySock) -> - [SrcAddrBin, DstAddrBin, SrcPortBin, DstPortBin] - = binary:split(ProxyInfo, [<<" ">>, <<"\r\n">>], [global, trim]), - {ok, SrcAddr} = inet:parse_address(binary_to_list(SrcAddrBin)), - {ok, DstAddr} = inet:parse_address(binary_to_list(DstAddrBin)), - SrcPort = list_to_integer(binary_to_list(SrcPortBin)), - DstPort = list_to_integer(binary_to_list(DstPortBin)), - {ok, ProxySock#proxy_socket{src_addr = SrcAddr, dst_addr = DstAddr, - src_port = SrcPort, dst_port = DstPort}}. - -parse_v2(?LOCAL, _Trans, _ProxyInfo, #proxy_socket{socket = Sock}) -> - {ok, Sock}; - -parse_v2(?PROXY, ?STREAM, ProxyInfo, ProxySock = #proxy_socket{inet = inet4}) -> - <> = ProxyInfo, - parse_pp2_additional(AdditionalBytes, ProxySock#proxy_socket{ - src_addr = {A, B, C, D}, src_port = SrcPort, - dst_addr = {W, X, Y, Z}, dst_port = DstPort}); - -parse_v2(?PROXY, ?STREAM, ProxyInfo, ProxySock = #proxy_socket{inet = inet6}) -> - <> = ProxyInfo, - parse_pp2_additional(AdditionalBytes, ProxySock#proxy_socket{ - src_addr = {A, B, C, D, E, F, G, H}, src_port = SrcPort, - dst_addr = {R, S, T, U, V, W, X, Y}, dst_port = DstPort}); - -parse_v2(_, _, _, #proxy_socket{socket = _Sock}) -> - {error, unsupported_proto_v2}. - -parse_pp2_additional(<<>>, ProxySock) -> - {ok, ProxySock}; -parse_pp2_additional(Bytes, ProxySock) when is_binary(Bytes) -> - IgnoreGuard = fun(?PP2_TYPE_NOOP) -> false; (_Type) -> true end, - AdditionalInfo = parse_pp2_tlv(fun pp2_additional_field/1, Bytes, IgnoreGuard), - {ok, ProxySock#proxy_socket{pp2_additional_info = AdditionalInfo}}. - -parse_pp2_tlv(Fun, Bytes) -> - parse_pp2_tlv(Fun, Bytes, fun(_Any) -> true end). -parse_pp2_tlv(Fun, Bytes, Guard) -> - [Fun({Type, Val}) || <> <= Bytes, Guard(Type)]. - -pp2_additional_field({?PP2_TYPE_ALPN, PP2_ALPN}) -> - {pp2_alpn, PP2_ALPN}; -pp2_additional_field({?PP2_TYPE_AUTHORITY, PP2_AUTHORITY}) -> - {pp2_authority, PP2_AUTHORITY}; -pp2_additional_field({?PP2_TYPE_CRC32C, PP2_CRC32C}) -> - {pp2_crc32c, PP2_CRC32C}; -pp2_additional_field({?PP2_TYPE_NETNS, PP2_NETNS}) -> - {pp2_netns, PP2_NETNS}; -pp2_additional_field({?PP2_TYPE_SSL, PP2_SSL}) -> - {pp2_ssl, parse_pp2_ssl(PP2_SSL)}; -pp2_additional_field({Field, Value}) -> - {{pp2_raw, Field}, Value}. - -parse_pp2_ssl(<<_Unused:5, PP2_CLIENT_CERT_SESS:1, PP2_CLIENT_CERT_CONN:1, PP2_CLIENT_SSL:1, - PP2_SSL_VERIFY:32, SubFields/bitstring>>) -> - [ - %% The PP2_CLIENT_SSL flag indicates that the client connected over SSL/TLS. When - %% this field is present, the US-ASCII string representation of the TLS version is - %% appended at the end of the field in the TLV format using the type PP2_SUBTYPE_SSL_VERSION. - {pp2_ssl_client, bool(PP2_CLIENT_SSL)}, - - %% PP2_CLIENT_CERT_CONN indicates that the client provided a certificate over the - %% current connection. - {pp2_ssl_client_cert_conn, bool(PP2_CLIENT_CERT_CONN)}, - - %% PP2_CLIENT_CERT_SESS indicates that the client provided a - %% certificate at least once over the TLS session this connection belongs to. - {pp2_ssl_client_cert_sess, bool(PP2_CLIENT_CERT_SESS)}, - - %% The field will be zero if the client presented a certificate - %% and it was successfully verified, and non-zero otherwise. - {pp2_ssl_verify, ssl_certificate_verified(PP2_SSL_VERIFY)} - - | parse_pp2_tlv(fun pp2_additional_ssl_field/1, SubFields) - ]. - -pp2_additional_ssl_field({?PP2_SUBTYPE_SSL_VERSION, PP2_SSL_VERSION}) -> - {pp2_ssl_version, PP2_SSL_VERSION}; - -%% In all cases, the string representation (in UTF8) of the Common Name field -%% (OID: 2.5.4.3) of the client certificate's Distinguished Name, is appended -%% using the TLV format and the type PP2_SUBTYPE_SSL_CN. E.g. "example.com". -pp2_additional_ssl_field({?PP2_SUBTYPE_SSL_CN, PP2_SSL_CN}) -> - {pp2_ssl_cn, PP2_SSL_CN}; -pp2_additional_ssl_field({?PP2_SUBTYPE_SSL_CIPHER, PP2_SSL_CIPHER}) -> - {pp2_ssl_cipher, PP2_SSL_CIPHER}; -pp2_additional_ssl_field({?PP2_SUBTYPE_SSL_SIG_ALG, PP2_SSL_SIG_ALG}) -> - {pp2_ssl_sig_alg, PP2_SSL_SIG_ALG}; -pp2_additional_ssl_field({?PP2_SUBTYPE_SSL_KEY_ALG, PP2_SSL_KEY_ALG}) -> - {pp2_ssl_key_alg, PP2_SSL_KEY_ALG}; -pp2_additional_ssl_field({Field, Val}) -> - {{pp2_ssl_raw, Field}, Val}. - -ssl_certificate_verified(0) -> success; -ssl_certificate_verified(_) -> failed. - -%% V1 -inet_family($4) -> inet4; -inet_family($6) -> inet6; - -%% V2 -inet_family(?UNSPEC) -> unspec; -inet_family(?INET) -> inet4; -inet_family(?INET6) -> inet6; -inet_family(?UNIX) -> unix. - -bool(1) -> true; -bool(_) -> false. - diff --git a/src/proxyPt/ntPptAcceptor.erl b/src/proxyPt/ntPptAcceptor.erl index e5e89c6..a2fc715 100644 --- a/src/proxyPt/ntPptAcceptor.erl +++ b/src/proxyPt/ntPptAcceptor.erl @@ -2,14 +2,15 @@ -include("eNet.hrl"). -include("ntCom.hrl"). +-include("proxyPt.hrl"). -compile(inline). -compile({inline_size, 128}). -export([ - start_link/5 + start_link/7 - , handshake/3 + , pptAndHS/5 , init/1 , handleMsg/2 @@ -22,9 +23,9 @@ ]). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% genActor start %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% --spec start_link(list(), timeout(), socket(), module(), [proc_lib:spawn_option()]) -> {ok, pid()}. -start_link(SslOpts, HandshakeTimeout, LSock, ConMod, SpawnOpts) -> - proc_lib:start_link(?MODULE, init_it, [self(), {SslOpts, HandshakeTimeout, LSock, ConMod}], infinity, SpawnOpts). +-spec start_link(list(), timeout(), boolean(), timeout(), inet:socket(), module(), [proc_lib:spawn_option()]) -> {ok, pid()}. +start_link(SslOpts, SslHSTet, ProxyPt, ProxyPtTet, LSock, ConMod, SpawnOpts) -> + proc_lib:start_link(?MODULE, init_it, [self(), {SslOpts, SslHSTet, ProxyPt, ProxyPtTet, LSock, ConMod}], infinity, SpawnOpts). init_it(Parent, Args) -> process_flag(trap_exit, true), @@ -75,24 +76,26 @@ loop(Parent, State) -> -record(state, { lSock , sslOpts - , handshake_timeout + , sslHSTet + , proxyPt + , proxyPtTet , ref , conMod , sockMod }). -spec init(Args :: term()) -> ok. -init({SslOpts, HandshakeTimeout, LSock, ConMod}) -> +init({SslOpts, SslHSTet, ProxyPt, ProxyPtTet, LSock, ConMod}) -> case prim_inet:async_accept(LSock, -1) of {ok, Ref} -> {ok, SockMod} = inet_db:lookup_socket(LSock), - {ok, #state{lSock = LSock, sslOpts = SslOpts, handshake_timeout = HandshakeTimeout, ref = Ref, conMod = ConMod, sockMod = SockMod}}; + {ok, #state{lSock = LSock, sslOpts = SslOpts, sslHSTet = SslHSTet, proxyPt = ProxyPt, proxyPtTet = ProxyPtTet, ref = Ref, conMod = ConMod, sockMod = SockMod}}; {error, Reason} -> ?ntErr("init prim_inet:async_accept error ~p~n", [Reason]), {stop, Reason} end. -handleMsg({inet_async, LSock, Ref, Msg}, #state{lSock = LSock, sslOpts = SslOpts, handshake_timeout = HandshakeTimeout, ref = Ref, conMod = ConMod, sockMod = SockMod} = State) -> +handleMsg({inet_async, LSock, Ref, Msg}, #state{lSock = LSock, sslOpts = SslOpts, sslHSTet = SslHSTet, proxyPt = ProxyPt, proxyPtTet = ProxyPtTet, ref = Ref, conMod = ConMod, sockMod = SockMod} = State) -> case Msg of {ok, Sock} -> %% make it look like gen_tcp:accept @@ -100,7 +103,7 @@ handleMsg({inet_async, LSock, Ref, Msg}, #state{lSock = LSock, sslOpts = SslOpts try ConMod:newConn(Sock) of {ok, Pid} -> gen_tcp:controlling_process(Sock, Pid), - Pid ! {?mSockReady, Sock, SslOpts, HandshakeTimeout}, + Pid ! {?mSockReady, Sock, SslOpts, SslHSTet, ProxyPt, ProxyPtTet}, newAsyncAccept(LSock, State); {close, Reason} -> ?ntErr("handleMsg ConMod:newAcceptor return close ~p~n", [Reason]), @@ -134,11 +137,31 @@ newAsyncAccept(LSock, State) -> {stop, Reason} end. -handshake(Sock, SslOpts, Timeout) -> - case ssl:handshake(Sock, SslOpts, Timeout) of - {ok, _SslSock} = Ret -> - Ret; - {ok, SslSock, _Ext} -> %% OTP 21.0 - {ok, SslSock}; - {error, _} = Err -> Err +pptAndHS(OSock, SslOpts, SslHSTet, ProxyPt, ProxyPtTet) -> + PptRet = + case ProxyPt of + true -> + nt_proxy_protocol:recv(OSock, ProxyPtTet); + _ -> + {ok, OSock, #proxy_socket{}} + end, + + case PptRet of + {ok, TemSock, ProxySock} -> + case SslOpts /= undefined of + true -> + case ssl:handshake(TemSock, SslOpts, SslHSTet) of + {ok, SslSock} -> + {ok, SslSock, ProxySock}; + {ok, SslSock, _Ext} -> %% OTP 21.0 + {ok, SslSock, ProxySock}; + {error, _} = Err -> Err + end; + _ -> + PptRet + end; + _ -> + PptRet end. + + diff --git a/src/proxyPt/ntPptAcceptorSup.erl b/src/proxyPt/ntPptAcceptorSup.erl index 5f1a06c..4184eb8 100644 --- a/src/proxyPt/ntPptAcceptorSup.erl +++ b/src/proxyPt/ntPptAcceptorSup.erl @@ -3,23 +3,23 @@ -behaviour(supervisor). -export([ - start_link/3 + start_link/5 ]). -export([ init/1 ]). --spec(start_link(SupName :: atom(), SslOpts :: list(), HandshakeTimeout :: timeout()) -> {ok, pid()}). -start_link(SupName, SslOpts, HandshakeTimeout) -> - supervisor:start_link({local, SupName}, ?MODULE, {SslOpts, HandshakeTimeout}). +-spec(start_link(SupName :: atom(), SslOpts :: list(), SslHSTet :: timeout(), ProxyPt ::boolean(), ProxyPtTet ::timeout()) -> {ok, pid()}). +start_link(SupName, SslOpts, SslHSTet, ProxyPt, ProxyPtTet) -> + supervisor:start_link({local, SupName}, ?MODULE, {SslOpts, SslHSTet, ProxyPt, ProxyPtTet}). -init({SslOpts, HandshakeTimeout}) -> +init({SslOpts, SslHSTet, ProxyPt, ProxyPtTet}) -> SupFlags = #{strategy => simple_one_for_one, intensity => 100, period => 3600}, Acceptor = #{ id => ntPptAcceptor, - start => {ntPptAcceptor, start_link, [SslOpts, HandshakeTimeout]}, + start => {ntPptAcceptor, start_link, [SslOpts, SslHSTet, ProxyPt, ProxyPtTet]}, restart => transient, shutdown => 3000, type => worker, diff --git a/src/proxyPt/ntPptListener.erl b/src/proxyPt/ntPptListener.erl index 84fd532..6abaa4e 100644 --- a/src/proxyPt/ntPptListener.erl +++ b/src/proxyPt/ntPptListener.erl @@ -98,7 +98,7 @@ init({AptSupName, Port, ListenOpts}) -> %% Don't active the socket... case gen_tcp:listen(Port, lists:keystore(active, 1, LastTcpOpts, {active, false})) of {ok, LSock} -> - AptCnt = ?getLValue(aptCnt, ListenOpts, ?ACCEPTOR_POOL), + AptCnt = ?getLValue(aptCnt, ListenOpts, ?AptCnt), ConMod = ?getLValue(conMod, ListenOpts, undefined), startAcceptor(AptCnt, LSock, AptSupName, ConMod), {ok, {LAddr, LPort}} = inet:sockname(LSock), diff --git a/src/proxyPt/ntPptMgrSup.erl b/src/proxyPt/ntPptMgrSup.erl index 9473e62..eb74697 100644 --- a/src/proxyPt/ntPptMgrSup.erl +++ b/src/proxyPt/ntPptMgrSup.erl @@ -23,19 +23,15 @@ init({SupName, Port, ListenOpts}) -> AptSupName = ntCom:asName(ssl, SupName), ListenName = ntCom:lsName(ssl, SupName), - SslOpts = ?getLValue(sslOpts, ListenOpts, []), - {HandshakeTimeout, LastSslOpts} = - case lists:keytake(handshake_timeout, 1, SslOpts) of - {value, {handshake_timeout, Timeout}, TemSslOpts} -> - {Timeout, TemSslOpts}; - false -> - {?SSL_HANDSHAKE_TIMEOUT, SslOpts} - end, + SslOpts = ?getLValue(sslOpts, ListenOpts, undefined), + SslHSTet = ?getLValue(sslHSTet, ListenOpts, ?DefSslHSTet), + ProxyPt = ?getLValue(proxyPt, ListenOpts, false), + ProxyPtTet = ?getLValue(proxyPtTet, ListenOpts, ?DefProxyPtTet), ChildSpecs = [ #{ id => AptSupName, - start => {ntPptAcceptorSup, start_link, [AptSupName, LastSslOpts, HandshakeTimeout]}, + start => {ntPptAcceptorSup, start_link, [AptSupName, SslOpts, SslHSTet, ProxyPt, ProxyPtTet]}, restart => permanent, shutdown => infinity, type => supervisor, diff --git a/src/proxyPt/nt_proxy_protocol.erl b/src/proxyPt/nt_proxy_protocol.erl new file mode 100644 index 0000000..1cdf45f --- /dev/null +++ b/src/proxyPt/nt_proxy_protocol.erl @@ -0,0 +1,197 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @doc [Proxy Protocol](https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt) +-module(nt_proxy_protocol). + +-include("proxyPt.hrl"). + +-export([recv/2]). + +-ifdef(TEST). +-export([parse_v1/3, parse_v2/5, parse_pp2_tlv/2, parse_pp2_ssl/1]). +-endif. + +%% Protocol Command +-define(LOCAL, 16#0). +-define(PROXY, 16#1). + +%% Address families +-define(UNSPEC, 16#0). +-define(INET, 16#1). +-define(INET6, 16#2). +-define(UNIX, 16#3). + +-define(STREAM, 16#1). +-define(DGRAM, 16#2). + +-define(SPACE, 16#20). + +-define(TIMEOUT, 5000). + +%% Proxy Protocol Additional Fields +-define(PP2_TYPE_ALPN, 16#01). +-define(PP2_TYPE_AUTHORITY, 16#02). +-define(PP2_TYPE_CRC32C, 16#03). +-define(PP2_TYPE_NOOP, 16#04). +-define(PP2_TYPE_SSL, 16#20). +-define(PP2_SUBTYPE_SSL_VERSION, 16#21). +-define(PP2_SUBTYPE_SSL_CN, 16#22). +-define(PP2_SUBTYPE_SSL_CIPHER, 16#23). +-define(PP2_SUBTYPE_SSL_SIG_ALG, 16#24). +-define(PP2_SUBTYPE_SSL_KEY_ALG, 16#25). +-define(PP2_TYPE_NETNS, 16#30). + +%% Protocol signature: +%% 16#0D,16#0A,16#00,16#0D,16#0A,16#51,16#55,16#49,16#54,16#0A +-define(SIG, "\r\n\0\r\nQUIT\n"). + +-spec(recv(inet:socket(), timeout()) -> {ok, inet:socket(), #proxy_socket{}} | {error, term()}). +recv(Sock, Timeout) -> + {ok, OriginOpts} = inet:getopts(Sock, [mode, active, packet]), + ok = inet:setopts(Sock, [binary, {active, once}, {packet, line}]), + receive + %% V1 TCP + {_, Sock, <<"PROXY TCP", Proto, ?SPACE, ProxyInfo/binary>>} -> + inet:setopts(Sock, OriginOpts), + parse_v1(Sock, ProxyInfo, #proxy_socket{inet = inet_family(Proto)}); + %% V1 Unknown + {_, _Sock, <<"PROXY UNKNOWN", _ProxyInfo/binary>>} -> + inet:setopts(Sock, OriginOpts), + {ok, Sock, #proxy_socket{}}; + %% V2 TCP + {_, _Sock, <<"\r\n">>} -> + inet:setopts(Sock, [{active, false}, {packet, raw}]), + {ok, Header} = gen_tcp:recv(Sock, 14, 1000), + <> = Header, + case gen_tcp:recv(Sock, Len, 1000) of + {ok, ProxyInfo} -> + inet:setopts(Sock, OriginOpts), + parse_v2(Cmd, Trans, ProxyInfo, #proxy_socket{inet = inet_family(AF)}, Sock); + {error, Reason} -> + {error, {recv_proxy_info_error, Reason}} + end; + {tcp_error, _Sock, Reason} -> + {error, {recv_proxy_info_error, Reason}}; + {tcp_closed, _Sock} -> + {error, {recv_proxy_info_error, tcp_closed}}; + {_, _Sock, ProxyInfo} -> + {error, {invalid_proxy_info, ProxyInfo}} + after + Timeout -> + {error, proxy_proto_timeout} + end. + +parse_v1(Sock, ProxyInfo, ProxySock) -> + [SrcAddrBin, DstAddrBin, SrcPortBin, DstPortBin] = binary:split(ProxyInfo, [<<" ">>, <<"\r\n">>], [global, trim]), + {ok, SrcAddr} = inet:parse_address(binary_to_list(SrcAddrBin)), + {ok, DstAddr} = inet:parse_address(binary_to_list(DstAddrBin)), + SrcPort = list_to_integer(binary_to_list(SrcPortBin)), + DstPort = list_to_integer(binary_to_list(DstPortBin)), + {ok, Sock, ProxySock#proxy_socket{src_addr = SrcAddr, dst_addr = DstAddr, src_port = SrcPort, dst_port = DstPort}}. + +parse_v2(?LOCAL, _Trans, _ProxyInfo, ProxySocket , Sock) -> + {ok, Sock, ProxySocket}; +parse_v2(?PROXY, ?STREAM, ProxyInfo, ProxySock = #proxy_socket{inet = inet4}, Sock) -> + <> = ProxyInfo, + parse_pp2_additional(AdditionalBytes, ProxySock#proxy_socket{src_addr = {A, B, C, D}, src_port = SrcPort, dst_addr = {W, X, Y, Z}, dst_port = DstPort}, Sock); +parse_v2(?PROXY, ?STREAM, ProxyInfo, ProxySock = #proxy_socket{inet = inet6}, Sock) -> + <> = ProxyInfo, + parse_pp2_additional(AdditionalBytes, ProxySock#proxy_socket{src_addr = {A, B, C, D, E, F, G, H}, src_port = SrcPort, dst_addr = {R, S, T, U, V, W, X, Y}, dst_port = DstPort}, Sock); +parse_v2(_, _, _,_, _) -> + {error, unsupported_proto_v2}. + +parse_pp2_additional(<<>>, ProxySock, Sock) -> + {ok, Sock, ProxySock}; +parse_pp2_additional(Bytes, ProxySock, Sock) when is_binary(Bytes) -> + IgnoreGuard = fun(?PP2_TYPE_NOOP) -> false; (_Type) -> true end, + AdditionalInfo = parse_pp2_tlv(fun pp2_additional_field/1, Bytes, IgnoreGuard), + {ok, Sock, ProxySock#proxy_socket{pp2_additional_info = AdditionalInfo}}. + +parse_pp2_tlv(Fun, Bytes) -> + parse_pp2_tlv(Fun, Bytes, fun(_Any) -> true end). +parse_pp2_tlv(Fun, Bytes, Guard) -> + [Fun({Type, Val}) || <> <= Bytes, Guard(Type)]. + +pp2_additional_field({?PP2_TYPE_ALPN, PP2_ALPN}) -> + {pp2_alpn, PP2_ALPN}; +pp2_additional_field({?PP2_TYPE_AUTHORITY, PP2_AUTHORITY}) -> + {pp2_authority, PP2_AUTHORITY}; +pp2_additional_field({?PP2_TYPE_CRC32C, PP2_CRC32C}) -> + {pp2_crc32c, PP2_CRC32C}; +pp2_additional_field({?PP2_TYPE_NETNS, PP2_NETNS}) -> + {pp2_netns, PP2_NETNS}; +pp2_additional_field({?PP2_TYPE_SSL, PP2_SSL}) -> + {pp2_ssl, parse_pp2_ssl(PP2_SSL)}; +pp2_additional_field({Field, Value}) -> + {{pp2_raw, Field}, Value}. + +parse_pp2_ssl(<<_Unused:5, PP2_CLIENT_CERT_SESS:1, PP2_CLIENT_CERT_CONN:1, PP2_CLIENT_SSL:1, + PP2_SSL_VERIFY:32, SubFields/bitstring>>) -> + [ + %% The PP2_CLIENT_SSL flag indicates that the client connected over SSL/TLS. When + %% this field is present, the US-ASCII string representation of the TLS version is + %% appended at the end of the field in the TLV format using the type PP2_SUBTYPE_SSL_VERSION. + {pp2_ssl_client, bool(PP2_CLIENT_SSL)}, + + %% PP2_CLIENT_CERT_CONN indicates that the client provided a certificate over the + %% current connection. + {pp2_ssl_client_cert_conn, bool(PP2_CLIENT_CERT_CONN)}, + + %% PP2_CLIENT_CERT_SESS indicates that the client provided a + %% certificate at least once over the TLS session this connection belongs to. + {pp2_ssl_client_cert_sess, bool(PP2_CLIENT_CERT_SESS)}, + + %% The field will be zero if the client presented a certificate + %% and it was successfully verified, and non-zero otherwise. + {pp2_ssl_verify, ssl_certificate_verified(PP2_SSL_VERIFY)} + + | parse_pp2_tlv(fun pp2_additional_ssl_field/1, SubFields) + ]. + +pp2_additional_ssl_field({?PP2_SUBTYPE_SSL_VERSION, PP2_SSL_VERSION}) -> + {pp2_ssl_version, PP2_SSL_VERSION}; + +%% In all cases, the string representation (in UTF8) of the Common Name field +%% (OID: 2.5.4.3) of the client certificate's Distinguished Name, is appended +%% using the TLV format and the type PP2_SUBTYPE_SSL_CN. E.g. "example.com". +pp2_additional_ssl_field({?PP2_SUBTYPE_SSL_CN, PP2_SSL_CN}) -> + {pp2_ssl_cn, PP2_SSL_CN}; +pp2_additional_ssl_field({?PP2_SUBTYPE_SSL_CIPHER, PP2_SSL_CIPHER}) -> + {pp2_ssl_cipher, PP2_SSL_CIPHER}; +pp2_additional_ssl_field({?PP2_SUBTYPE_SSL_SIG_ALG, PP2_SSL_SIG_ALG}) -> + {pp2_ssl_sig_alg, PP2_SSL_SIG_ALG}; +pp2_additional_ssl_field({?PP2_SUBTYPE_SSL_KEY_ALG, PP2_SSL_KEY_ALG}) -> + {pp2_ssl_key_alg, PP2_SSL_KEY_ALG}; +pp2_additional_ssl_field({Field, Val}) -> + {{pp2_ssl_raw, Field}, Val}. + +ssl_certificate_verified(0) -> success; +ssl_certificate_verified(_) -> failed. + +%% V1 +inet_family($4) -> inet4; +inet_family($6) -> inet6; + +%% V2 +inet_family(?UNSPEC) -> unspec; +inet_family(?INET) -> inet4; +inet_family(?INET6) -> inet6; +inet_family(?UNIX) -> unix. + +bool(1) -> true; +bool(_) -> false. + diff --git a/src/ssl/ntSslAcceptor.erl b/src/ssl/ntSslAcceptor.erl index 0211864..9753dde 100644 --- a/src/ssl/ntSslAcceptor.erl +++ b/src/ssl/ntSslAcceptor.erl @@ -22,7 +22,7 @@ ]). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% genActor start %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% --spec start_link(list(), timeout(), socket(), module(), [proc_lib:spawn_option()]) -> {ok, pid()}. +-spec start_link(list(), timeout(), inet:socket(), module(), [proc_lib:spawn_option()]) -> {ok, pid()}. start_link(SslOpts, HandshakeTimeout, LSock, ConMod, SpawnOpts) -> proc_lib:start_link(?MODULE, init_it, [self(), {SslOpts, HandshakeTimeout, LSock, ConMod}], infinity, SpawnOpts). diff --git a/src/ssl/ntSslListener.erl b/src/ssl/ntSslListener.erl index dfb62ae..50a887f 100644 --- a/src/ssl/ntSslListener.erl +++ b/src/ssl/ntSslListener.erl @@ -98,7 +98,7 @@ init({AptSupName, Port, ListenOpts}) -> %% Don't active the socket... case gen_tcp:listen(Port, lists:keystore(active, 1, LastTcpOpts, {active, false})) of {ok, LSock} -> - AptCnt = ?getLValue(aptCnt, ListenOpts, ?ACCEPTOR_POOL), + AptCnt = ?getLValue(aptCnt, ListenOpts, ?AptCnt), ConMod = ?getLValue(conMod, ListenOpts, undefined), startAcceptor(AptCnt, LSock, AptSupName, ConMod), {ok, {LAddr, LPort}} = inet:sockname(LSock), diff --git a/src/ssl/ntSslMgrSup.erl b/src/ssl/ntSslMgrSup.erl index 6e678f8..c9707ba 100644 --- a/src/ssl/ntSslMgrSup.erl +++ b/src/ssl/ntSslMgrSup.erl @@ -24,7 +24,7 @@ init({SupName, Port, ListenOpts}) -> ListenName = ntCom:lsName(ssl, SupName), SslOpts = ?getLValue(sslOpts, ListenOpts, []), - SslHSTet = ?getLValue(sslHSTet, ListenOpts, ?SSL_HANDSHAKE_TIMEOUT), + SslHSTet = ?getLValue(sslHSTet, ListenOpts, ?DefSslHSTet), ChildSpecs = [ #{ diff --git a/src/tcp/ntTcpAcceptor.erl b/src/tcp/ntTcpAcceptor.erl index 946cb33..d0c9ca6 100644 --- a/src/tcp/ntTcpAcceptor.erl +++ b/src/tcp/ntTcpAcceptor.erl @@ -20,7 +20,7 @@ ]). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% genActor start %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% --spec start_link(socket(), module(), [proc_lib:spawn_option()]) -> {ok, pid()}. +-spec start_link(inet:socket(), module(), [proc_lib:spawn_option()]) -> {ok, pid()}. start_link(LSock, ConMod, SpawnOpts) -> proc_lib:start_link(?MODULE, init_it, [self(), {LSock, ConMod}], infinity, SpawnOpts). diff --git a/src/tcp/ntTcpListener.erl b/src/tcp/ntTcpListener.erl index d10f19b..fd58ffc 100644 --- a/src/tcp/ntTcpListener.erl +++ b/src/tcp/ntTcpListener.erl @@ -98,7 +98,7 @@ init({AptSupName, Port, ListenOpts}) -> %% Don't active the socket... case gen_tcp:listen(Port, lists:keystore(active, 1, LastTcpOpts, {active, false})) of {ok, LSock} -> - AptCnt = ?getLValue(aptCnt, ListenOpts, ?ACCEPTOR_POOL), + AptCnt = ?getLValue(aptCnt, ListenOpts, ?AptCnt), ConMod = ?getLValue(conMod, ListenOpts, undefined), startAcceptor(AptCnt, LSock, AptSupName, ConMod), {ok, {LAddr, LPort}} = inet:sockname(LSock), diff --git a/src/udp/ntUdpSrv.erl b/src/udp/ntUdpSrv.erl index 487a81f..b71f8ba 100644 --- a/src/udp/ntUdpSrv.erl +++ b/src/udp/ntUdpSrv.erl @@ -102,7 +102,7 @@ init({Port, UoOpts}) -> %% Don't active the socket... case gen_udp:open(Port, lists:keystore(active, 1, LastUdpOpts, {active, false})) of {ok, OSock} -> - AptCnt = ?getLValue(aptCnt, UoOpts, ?ACCEPTOR_POOL), + AptCnt = ?getLValue(aptCnt, UoOpts, ?AptCnt), ConMod = ?getLValue(conMod, UoOpts, undefined), {ok, {LAddr, LPort}} = inet:sockname(OSock), ?ntInfo("success to open on ~p ~p ~n", [LAddr, LPort]),