diff --git a/.gitignore b/.gitignore index 6e8057b..4dc0202 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,6 @@ ebin/ doc/*.html doc/*.css doc/*.png -doc/edoc-info \ No newline at end of file +doc/edoc-info +Emakefile +*.bat \ No newline at end of file diff --git a/README.md b/README.md index 0af5161..89ca67c 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,7 @@ ibrowse is a HTTP client written in erlang. * Asynchronous requests. Responses are streamed to a process * Basic authentication * Supports proxy authentication +* Supports socks5 * Can talk to secure webservers using SSL * *Any other features in the code not listed here :)* @@ -279,3 +280,17 @@ support this. Nor did www.google.com. But good old BBC supports this: {"Via","1.1 hatproxy01 (NetCache NetApp/5.6.2)"}], "TRACE / HTTP/1.1\r\nHost: www.bbc.co.uk\r\nConnection: keep-alive\r\nX-Forwarded-For: 172.24.28.29\r\nVia: 1.1 hatproxy01 (NetCache NetApp/5.6.2)\r\nCookie: BBC-UID=7452e...\r\n\r\n"} ``` + +A `GET` using a socks5: + +```erlang +ibrowse:send_req("http://google.com", [], get, [], + [{socks5_host, "127.0.0.1"}, + {socks5_port, 5335}]). + +ibrowse:send_req("http://google.com", [], get, [], + [{socks5_host, "127.0.0.1"}, + {socks5_port, 5335}, + {socks5_user, "user4321"}, + {socks5_pass, "pass7654"}]). +``` diff --git a/src/ibrowse.erl b/src/ibrowse.erl index 5364587..42030af 100644 --- a/src/ibrowse.erl +++ b/src/ibrowse.erl @@ -443,6 +443,8 @@ do_send_req(Conn_Pid, Parsed_url, Headers, Method, Body, Options, Timeout) -> {error, sel_conn_closed}; {'EXIT', {normal, _}} -> {error, req_timedout}; + {'EXIT', {connection_closed, _}} -> + {error, sel_conn_closed}; {error, connection_closed} -> {error, sel_conn_closed}; {'EXIT', Reason} -> diff --git a/src/ibrowse_http_client.erl b/src/ibrowse_http_client.erl index b014e92..fd2c25d 100644 --- a/src/ibrowse_http_client.erl +++ b/src/ibrowse_http_client.erl @@ -215,11 +215,11 @@ handle_info({stream_close, _Req_id}, State) -> handle_info({tcp_closed, _Sock}, State) -> do_trace("TCP connection closed by peer!~n", []), handle_sock_closed(State), - {stop, normal, State}; + {stop, connection_closed, State}; handle_info({ssl_closed, _Sock}, State) -> do_trace("SSL connection closed by peer!~n", []), handle_sock_closed(State), - {stop, normal, State}; + {stop, connection_closed, State}; handle_info({tcp_error, _Sock, Reason}, State) -> do_trace("Error on connection to ~1000.p:~1000.p -> ~1000.p~n", @@ -502,7 +502,13 @@ do_connect(Host, Port, Options, #state{is_ssl = true, Timeout) -> ssl:connect(Host, Port, get_sock_options(Host, Options, SSLOptions), Timeout); do_connect(Host, Port, Options, _State, Timeout) -> - gen_tcp:connect(Host, Port, get_sock_options(Host, Options, []), Timeout). + Socks5Host = get_value(socks5_host, Options, undefined), + case Socks5Host of + undefined -> + gen_tcp:connect(Host, Port, get_sock_options(Host, Options, []), Timeout); + _ -> + catch ibrowse_socks5:connect(Host, Port, Options) + end. get_sock_options(Host, Options, SSLOptions) -> Caller_socket_options = get_value(socket_options, Options, []), @@ -1045,7 +1051,9 @@ parse_response(Data, #state{reply_buffer = Acc, reqs = Reqs, http_status_code=StatCode} end, put(conn_close, ConnClose), - TransferEncoding = to_lower(get_value("transfer-encoding", LCHeaders, "false")), + TransferEncodings = to_lower(get_value("transfer-encoding", LCHeaders, "false")), + IsChunked = lists:any(fun(Enc) -> string:strip(Enc) =:= "chunked" end, + string:tokens(TransferEncodings, ",")), Head_response_with_body = lists:member({workaround, head_response_with_body}, Options), case get_value("content-length", LCHeaders, undefined) of _ when Method == connect, @@ -1096,7 +1104,7 @@ parse_response(Data, #state{reply_buffer = Acc, reqs = Reqs, State_2 = reset_state(State_1_1), State_3 = set_cur_request(State_2#state{reqs = Reqs_1}), parse_response(Data_1, State_3); - _ when TransferEncoding =:= "chunked" -> + _ when IsChunked -> do_trace("Chunked encoding detected...~n",[]), send_async_headers(ReqId, StreamTo, Give_raw_headers, State_1), case parse_11_response(Data_1, State_1#state{transfer_encoding=chunked, diff --git a/src/ibrowse_socks5.erl b/src/ibrowse_socks5.erl new file mode 100644 index 0000000..10d88c1 --- /dev/null +++ b/src/ibrowse_socks5.erl @@ -0,0 +1,57 @@ +-module(ibrowse_socks5). + +-include_lib("kernel/src/inet_dns.hrl"). + +-export([connect/3]). + +-define(TIMEOUT, 2000). + +-define(SOCKS5, 5). +-define(AUTH_METHOD_NO, 0). +-define(AUTH_METHOD_USERPASS, 2). +-define(ADDRESS_TYPE_IP4, 1). +-define(COMMAND_TYPE_TCPIP_STREAM, 1). +-define(RESERVER, 0). +-define(STATUS_GRANTED, 0). + +-define(DNS_IP, {8,8,8,8}). + +connect(Host, Port, Options) -> + Socks5Host = proplists:get_value(socks5_host, Options), + Socks5Port = proplists:get_value(socks5_port, Options), + + {ok, Socket} = gen_tcp:connect(Socks5Host, Socks5Port, [binary, {packet, 0}, {keepalive, true}, {active, false}]), + + case proplists:get_value(socks5_user, Options, undefined) of + undefined -> + ok = gen_tcp:send(Socket, <>), + {ok, <>} = gen_tcp:recv(Socket, 2, ?TIMEOUT); + _Else -> + Socks5User = list_to_binary(proplists:get_value(socks5_user, Options)), + Socks5Pass = list_to_binary(proplists:get_value(socks5_pass, Options)), + + ok = gen_tcp:send(Socket, <>), + {ok, <>} = gen_tcp:recv(Socket, 2, ?TIMEOUT), + + UserLength = byte_size(Socks5User), + + ok = gen_tcp:send(Socket, << 1, UserLength >>), + ok = gen_tcp:send(Socket, Socks5User), + PassLength = byte_size(Socks5Pass), + ok = gen_tcp:send(Socket, << PassLength >>), + ok = gen_tcp:send(Socket, Socks5Pass), + {ok, <<1, 0>>} = gen_tcp:recv(Socket, 2, ?TIMEOUT) + end, + + {IP1,IP2,IP3,IP4} = case inet_parse:address(Host) of + {ok, IP} -> + IP; + _Other -> + {ok, NsData} = inet_res:nslookup(Host, in, a, [{?DNS_IP, 53}]), + [Addr | _NewAnList] = [D || #dns_rr{data=D, type=a} <- NsData#dns_rec.anlist], + Addr + end, + + ok = gen_tcp:send(Socket, <>), + {ok, << ?SOCKS5, ?STATUS_GRANTED, ?RESERVER, ?ADDRESS_TYPE_IP4, IP1, IP2, IP3, IP4, Port:16 >>} = gen_tcp:recv(Socket, 10, ?TIMEOUT), + {ok, Socket}.