Merge branch 'master' of https://github.com/cmullaparthi/ibrowse

.gitignore

@@ -5,4 +5,6 @@ ebin/
 doc/*.html
 doc/*.css
 doc/*.png
-doc/edoc-info
+doc/edoc-info
+Emakefile
+*.bat

README.md

@@ -27,6 +27,7 @@ ibrowse is a HTTP client written in erlang.
 *  Asynchronous requests. Responses are streamed to a process
 *  Basic authentication
 *  Supports proxy authentication
+*  Supports socks5
 *  Can talk to secure webservers using SSL
 *  *Any other features in the code not listed here :)*
 
@@ -279,3 +280,17 @@ support this. Nor did www.google.com. But good old BBC supports this:
      {"Via","1.1 hatproxy01 (NetCache NetApp/5.6.2)"}],
     "TRACE / HTTP/1.1\r\nHost: www.bbc.co.uk\r\nConnection: keep-alive\r\nX-Forwarded-For: 172.24.28.29\r\nVia: 1.1 hatproxy01 (NetCache NetApp/5.6.2)\r\nCookie: BBC-UID=7452e...\r\n\r\n"}
 ```
+
+A `GET` using a socks5:
+
+```erlang
+ibrowse:send_req("http://google.com", [], get, [],
+  [{socks5_host, "127.0.0.1"},
+  {socks5_port, 5335}]).
+
+ibrowse:send_req("http://google.com", [], get, [],
+  [{socks5_host, "127.0.0.1"},
+  {socks5_port, 5335},
+  {socks5_user, "user4321"},
+  {socks5_pass, "pass7654"}]).
+```

src/ibrowse.erl

@@ -443,6 +443,8 @@ do_send_req(Conn_Pid, Parsed_url, Headers, Method, Body, Options, Timeout) ->
             {error, sel_conn_closed};
         {'EXIT', {normal, _}} ->
             {error, req_timedout};
+        {'EXIT', {connection_closed, _}} ->
+            {error, sel_conn_closed};
         {error, connection_closed} ->
             {error, sel_conn_closed};
         {'EXIT', Reason} ->

src/ibrowse_http_client.erl

@@ -215,11 +215,11 @@ handle_info({stream_close, _Req_id}, State) ->
 handle_info({tcp_closed, _Sock}, State) ->
     do_trace("TCP connection closed by peer!~n", []),
     handle_sock_closed(State),
-    {stop, normal, State};
+    {stop, connection_closed, State};
 handle_info({ssl_closed, _Sock}, State) ->
     do_trace("SSL connection closed by peer!~n", []),
     handle_sock_closed(State),
-    {stop, normal, State};
+    {stop, connection_closed, State};
 
 handle_info({tcp_error, _Sock, Reason}, State) ->
     do_trace("Error on connection to ~1000.p:~1000.p -> ~1000.p~n",
@@ -502,7 +502,13 @@ do_connect(Host, Port, Options, #state{is_ssl      = true,
            Timeout) ->
     ssl:connect(Host, Port, get_sock_options(Host, Options, SSLOptions), Timeout);
 do_connect(Host, Port, Options, _State, Timeout) ->
-    gen_tcp:connect(Host, Port, get_sock_options(Host, Options, []), Timeout).
+    Socks5Host = get_value(socks5_host, Options, undefined),
+    case Socks5Host of
+      undefined ->
+        gen_tcp:connect(Host, Port, get_sock_options(Host, Options, []), Timeout);
+      _ ->
+        catch ibrowse_socks5:connect(Host, Port, Options)
+    end.
 
 get_sock_options(Host, Options, SSLOptions) ->
     Caller_socket_options = get_value(socket_options, Options, []),
@@ -1045,7 +1051,9 @@ parse_response(Data, #state{reply_buffer = Acc, reqs = Reqs,
                                             http_status_code=StatCode}
                       end,
             put(conn_close, ConnClose),
-            TransferEncoding = to_lower(get_value("transfer-encoding", LCHeaders, "false")),
+            TransferEncodings = to_lower(get_value("transfer-encoding", LCHeaders, "false")),
+            IsChunked = lists:any(fun(Enc) -> string:strip(Enc) =:= "chunked" end,
+                                  string:tokens(TransferEncodings, ",")),
             Head_response_with_body = lists:member({workaround, head_response_with_body}, Options),
             case get_value("content-length", LCHeaders, undefined) of
                 _ when Method == connect,
@@ -1096,7 +1104,7 @@ parse_response(Data, #state{reply_buffer = Acc, reqs = Reqs,
                     State_2 = reset_state(State_1_1),
                     State_3 = set_cur_request(State_2#state{reqs = Reqs_1}),
                     parse_response(Data_1, State_3);
-                _ when TransferEncoding =:= "chunked" ->
+                _ when IsChunked ->
                     do_trace("Chunked encoding detected...~n",[]),
                     send_async_headers(ReqId, StreamTo, Give_raw_headers, State_1),
                     case parse_11_response(Data_1, State_1#state{transfer_encoding=chunked,

src/ibrowse_socks5.erl

@@ -0,0 +1,57 @@
+-module(ibrowse_socks5).
+
+-include_lib("kernel/src/inet_dns.hrl").
+
+-export([connect/3]).
+
+-define(TIMEOUT, 2000).
+
+-define(SOCKS5, 5).
+-define(AUTH_METHOD_NO, 0).
+-define(AUTH_METHOD_USERPASS, 2).
+-define(ADDRESS_TYPE_IP4, 1).
+-define(COMMAND_TYPE_TCPIP_STREAM, 1).
+-define(RESERVER, 0).
+-define(STATUS_GRANTED, 0).
+
+-define(DNS_IP, {8,8,8,8}).
+
+connect(Host, Port, Options) ->
+    Socks5Host = proplists:get_value(socks5_host, Options),
+    Socks5Port = proplists:get_value(socks5_port, Options),
+
+    {ok, Socket} = gen_tcp:connect(Socks5Host, Socks5Port, [binary, {packet, 0}, {keepalive, true}, {active, false}]),
+
+    case proplists:get_value(socks5_user, Options, undefined) of
+        undefined ->
+            ok = gen_tcp:send(Socket, <<?SOCKS5, 1, ?AUTH_METHOD_NO>>),
+            {ok, <<?SOCKS5, ?AUTH_METHOD_NO>>} = gen_tcp:recv(Socket, 2, ?TIMEOUT);
+        _Else ->
+            Socks5User = list_to_binary(proplists:get_value(socks5_user, Options)),
+            Socks5Pass = list_to_binary(proplists:get_value(socks5_pass, Options)),
+
+            ok = gen_tcp:send(Socket, <<?SOCKS5, 1, ?AUTH_METHOD_USERPASS>>),
+            {ok, <<?SOCKS5, ?AUTH_METHOD_USERPASS>>} = gen_tcp:recv(Socket, 2, ?TIMEOUT),
+
+            UserLength = byte_size(Socks5User),
+
+            ok = gen_tcp:send(Socket, << 1, UserLength >>),
+            ok = gen_tcp:send(Socket, Socks5User),
+            PassLength = byte_size(Socks5Pass),
+            ok = gen_tcp:send(Socket, << PassLength >>),
+            ok = gen_tcp:send(Socket, Socks5Pass),
+            {ok, <<1, 0>>} = gen_tcp:recv(Socket, 2, ?TIMEOUT)
+    end,
+
+    {IP1,IP2,IP3,IP4} = case inet_parse:address(Host) of
+        {ok, IP} ->
+            IP;
+        _Other ->
+            {ok, NsData} = inet_res:nslookup(Host, in, a, [{?DNS_IP, 53}]),
+            [Addr | _NewAnList] = [D || #dns_rr{data=D, type=a} <- NsData#dns_rec.anlist],
+            Addr
+    end,
+
+    ok = gen_tcp:send(Socket, <<?SOCKS5, ?COMMAND_TYPE_TCPIP_STREAM, ?RESERVER, ?ADDRESS_TYPE_IP4, IP1, IP2, IP3, IP4, Port:16>>),
+    {ok, << ?SOCKS5, ?STATUS_GRANTED, ?RESERVER, ?ADDRESS_TYPE_IP4, IP1, IP2, IP3, IP4, Port:16 >>} = gen_tcp:recv(Socket, 10, ?TIMEOUT),
+    {ok, Socket}.