Bug fixes

.gitignore

@@ -10,3 +10,4 @@ mime.types
 .rebar
 *.plt
 .rebar
+*~

CHANGELOG

@@ -1,6 +1,12 @@
 CONTRIBUTIONS & CHANGE HISTORY
 ==============================
 
+19-04-2016 - v4.2.3
+             * Fix for https://github.com/cmullaparthi/ibrowse/issues/143
+             * Fix for https://github.com/cmullaparthi/ibrowse/issues/142
+             * Fix for https://github.com/cmullaparthi/ibrowse/issues/139
+             * Fixed behaviour of option preserve_status_line
+
 25-11-2015 - v4.2.2
              * Fix to ibrowse.app.src to enable publishing using Hex
 

CONTRIBUTORS

@@ -62,4 +62,4 @@ https://github.com/puzza007
 https://github.com/rflynn
 https://github.com/Vagabond
 https://github.com/divolgin
-
+https://github.com/vans163

Makefile

@@ -11,9 +11,9 @@ compile:
 	$(REBAR) compile
 
 clean:
-	$(REBAR) clean
+	@$(REBAR) clean && cd test && make clean && cd ..
 
-test: unit_tests old_tests eunit
+test: compile unit_tests eunit
 	@echo "====================================================="
 
 unit_tests:

README.md

@@ -7,7 +7,7 @@ ibrowse is a HTTP client written in erlang.
 
 **Comments to:** chandrashekhar.mullaparthi@gmail.com
 
-**Current Version:** 4.2.2
+**Current Version:** 4.2.3
 
 **Latest Version:** git://github.com/cmullaparthi/ibrowse.git
 
@@ -27,7 +27,8 @@ ibrowse is a HTTP client written in erlang.
 *  Asynchronous requests. Responses are streamed to a process
 *  Basic authentication
 *  Supports proxy authentication
-*  Supports socks5
+*  Supports SOCKS5
+    * Authentication methods 0 (No authentication) and 2(Username/password) supported
 *  Can talk to secure webservers using SSL
 *  *Any other features in the code not listed here :)*
 
@@ -292,5 +293,5 @@ ibrowse:send_req("http://google.com", [], get, [],
   [{socks5_host, "127.0.0.1"},
   {socks5_port, 5335},
   {socks5_user, "user4321"},
-  {socks5_pass, "pass7654"}]).
+  {socks5_password, "pass7654"}]).
 ```

src/ibrowse.app.src

@@ -1,6 +1,6 @@
 {application, ibrowse,
         [{description, "Erlang HTTP client application"},
-         {vsn, "4.2.2"},
+         {vsn, "4.2.3"},
          {registered, [ibrowse_sup, ibrowse]},
          {applications, [kernel,stdlib]},
 	 {env, []},

src/ibrowse.erl

@@ -300,8 +300,23 @@ send_req(Url, Headers, Method, Body) ->
 %%          {workaround, head_response_with_body}     |
 %%          {worker_process_options, list()} |
 %%          {return_raw_request, true}         |
-%%          {max_attempts, integer()}
+%%          {max_attempts, integer()}          |
+%%          {socks5_host, host()}              |
+%%          {socks5_port, integer()}           |
+%%          {socks5_user, binary()}            |
+%%          {socks5_password, binary()}
 %%
+%% ip4_address() = {0..255, 0..255, 0..255, 0..255}
+%% ip6_address() = 
+%%     {0..65535,
+%%      0..65535,
+%%      0..65535,
+%%      0..65535,
+%%      0..65535,
+%%      0..65535,
+%%      0..65535,
+%%      0..65535}
+%% host() = string() | ip4_address() | ip6_address()
 %% stream_to() = process() | {process(), once}
 %% process() = pid() | atom()
 %% username() = string()

src/ibrowse_http_client.erl

@@ -1174,6 +1174,7 @@ parse_response(Data, #state{reply_buffer = Acc, reqs = Reqs,
                                             http_status_code=StatCode};
                           false ->
                               State_0#state{recvd_headers=Headers_1, status=get_body,
+                                            status_line = Status_line,
                                             reply_buffer = <<>>,
                                             http_status_code=StatCode}
                       end,

src/ibrowse_socks5.erl

@@ -53,18 +53,29 @@ connect(Host, Port, Options, SockOptions, Timeout) ->
     end.
 
 handshake(Socket, Options) when is_port(Socket) ->
-    {Handshake, Success} = case get_value(socks5_user, Options, <<>>) of
-        <<>> ->
-            {<<?VERSION, 1, ?NO_AUTH>>, ?NO_AUTH};
-        User ->
-            Password = get_value(socks5_password, Options, <<>>),
-            {<<?VERSION, 1, ?USERPASS, (byte_size(User)), User,
-               (byte_size(Password)), Password>>, ?USERPASS}
-    end,
-    ok = gen_tcp:send(Socket, Handshake),
-    case gen_tcp:recv(Socket, 0) of
-        {ok, <<?VERSION, Success>>} ->
+    User = get_value(socks5_user, Options, <<>>),
+    Handshake_msg = case User of
+                        <<>> ->
+                            <<?VERSION, 1, ?NO_AUTH>>;
+                        User ->
+                            <<?VERSION, 1, ?USERPASS>>
+                    end,
+    ok = gen_tcp:send(Socket, Handshake_msg),
+    case gen_tcp:recv(Socket, 2) of
+        {ok, <<?VERSION, ?NO_AUTH>>} ->
             ok;
+        {ok, <<?VERSION, ?USERPASS>>} ->
+            Password = get_value(socks5_password, Options, <<>>),
+            Auth_msg = list_to_binary([1, 
+                                       iolist_size(User), User,
+                                       iolist_size(Password), Password]),
+            ok = gen_tcp:send(Socket, Auth_msg),
+            case gen_tcp:recv(Socket, 2) of
+                {ok, <<1, ?SUCCEEDED>>} ->
+                    ok;
+                _ ->
+                    {error, unacceptable}
+            end;
         {ok, <<?VERSION, ?UNACCEPTABLE>>} ->
             {error, unacceptable};
         {error, Reason} ->
@@ -76,18 +87,18 @@ connect(Host, Port, Via) when is_list(Host) ->
 connect(Host, Port, Via) when is_binary(Host), is_integer(Port),
                               is_port(Via) ->
     {AddressType, Address} = case inet:parse_address(binary_to_list(Host)) of
-        {ok, {IP1, IP2, IP3, IP4}} ->
-            {?ATYP_IPV4, <<IP1,IP2,IP3,IP4>>};
-        {ok, {IP1, IP2, IP3, IP4, IP5, IP6, IP7, IP8}} ->
-            {?ATYP_IPV6, <<IP1,IP2,IP3,IP4,IP5,IP6,IP7,IP8>>};
-        _ ->
-            HostLength = byte_size(Host),
-            {?ATYP_DOMAINNAME, <<HostLength,Host/binary>>}
-    end,
+                                 {ok, {IP1, IP2, IP3, IP4}} ->
+                                     {?ATYP_IPV4, <<IP1,IP2,IP3,IP4>>};
+                                 {ok, {IP1, IP2, IP3, IP4, IP5, IP6, IP7, IP8}} ->
+                                     {?ATYP_IPV6, <<IP1,IP2,IP3,IP4,IP5,IP6,IP7,IP8>>};
+                                 _ ->
+                                     HostLength = byte_size(Host),
+                                     {?ATYP_DOMAINNAME, <<HostLength,Host/binary>>}
+                             end,
     ok = gen_tcp:send(Via,
-        <<?VERSION, ?CONNECT, ?RESERVED,
-          AddressType, Address/binary,
-          (Port):16>>),
+                      <<?VERSION, ?CONNECT, ?RESERVED,
+                        AddressType, Address/binary,
+                        (Port):16>>),
     case gen_tcp:recv(Via, 0) of
         {ok, <<?VERSION, ?SUCCEEDED, ?RESERVED, _/binary>>} ->
             ok;

test/Makefile

@@ -1,13 +1,17 @@
 REBAR ?= $(shell which rebar3)
+IBROWSE_EBIN_PATH=../_build/default/lib/ibrowse/ebin
+
+compile:
+	@erl -pa $(IBROWSE_EBIN_PATH) -make
 
 test: compile
-	@erl -noshell -boot start_clean -pa ../../ibrowse/ebin -s ibrowse_test local_unit_tests -s erlang halt
+	@erl -noshell -boot start_clean -pa $(IBROWSE_EBIN_PATH) -s ibrowse_test local_unit_tests -s erlang halt
 
 old_tests: compile
-	@erl -noshell -boot start_clean -pa ../../ibrowse/ebin -s ibrowse_test unit_tests -s erlang halt
+	@erl -noshell -boot start_clean -pa $(IBROWSE_EBIN_PATH) -s ibrowse_test unit_tests -s erlang halt
 
-compile:
-	@erl -pa ../../ibrowse/ebin -make
+test_shell: compile
+	erl -boot start_clean -pa $(IBROWSE_EBIN_PATH) -s ibrowse_test_server start_server
 
 clean:
-	$(REBAR) clean
+	@rm -f *.beam

test/ibrowse_socks_server.erl

@@ -0,0 +1,218 @@
+%%%-------------------------------------------------------------------
+%%% @author Chandru Mullaparthi <>
+%%% @copyright (C) 2016, Chandru Mullaparthi
+%%% @doc
+%%%
+%%% @end
+%%% Created : 19 Apr 2016 by Chandru Mullaparthi <>
+%%%-------------------------------------------------------------------
+-module(ibrowse_socks_server).
+
+-behaviour(gen_server).
+
+%% API
+-export([start/2, stop/1]).
+
+%% gen_server callbacks
+-export([init/1, handle_call/3, handle_cast/2, handle_info/2,
+         terminate/2, code_change/3]).
+
+-define(SERVER, ?MODULE).
+
+-record(state, {listen_port, listen_socket, auth_method}).
+
+-define(NO_AUTH, 0).
+-define(AUTH_USER_PW, 2).
+
+%%%===================================================================
+%%% API
+%%%===================================================================
+
+start(Port, Auth_method) ->
+    Name = make_proc_name(Port),
+    gen_server:start({local, Name}, ?MODULE, [Port, Auth_method], []).
+
+stop(Port) ->
+    make_proc_name(Port) ! stop.
+
+make_proc_name(Port) ->
+    list_to_atom("ibrowse_socks_server_" ++ integer_to_list(Port)).
+
+%%%===================================================================
+%%% gen_server callbacks
+%%%===================================================================
+
+init([Port, Auth_method]) ->
+    State = #state{listen_port = Port, auth_method = Auth_method},
+    {ok, Sock} = gen_tcp:listen(State#state.listen_port, [{active, false}, binary, {reuseaddr, true}]),
+    self() ! accept_connection,
+    process_flag(trap_exit, true),
+    {ok, State#state{listen_socket = Sock}}.
+
+handle_call(_Request, _From, State) ->
+    Reply = ok,
+    {reply, Reply, State}.
+
+handle_cast(_Msg, State) ->
+    {noreply, State}.
+
+handle_info(accept_connection, State) ->
+    case gen_tcp:accept(State#state.listen_socket, 1000) of
+        {error, timeout} ->
+            self() ! accept_connection,
+            {noreply, State};
+        {ok, Socket} ->
+            Pid = proc_lib:spawn_link(fun() ->
+                                              socks_server_loop(Socket, State#state.auth_method)
+                                      end),
+            gen_tcp:controlling_process(Socket, Pid),
+            Pid ! ready,
+            self() ! accept_connection,
+            {noreply, State};
+        _Err ->
+            {stop, normal, State}
+    end;
+
+handle_info(stop, State) ->
+    {stop, normal, State};
+
+handle_info(_Info, State) ->
+    {noreply, State}.
+
+terminate(_Reason, _State) ->
+    ok.
+
+code_change(_OldVsn, State, _Extra) ->
+    {ok, State}.
+
+%%%===================================================================
+%%% Internal functions
+%%%===================================================================
+socks_server_loop(In_socket, Auth_method) ->
+    receive
+        ready ->
+            socks_server_loop(In_socket, Auth_method, <<>>, unauth)
+    end.
+
+socks_server_loop(In_socket, Auth_method, Acc, unauth) ->
+    inet:setopts(In_socket, [{active, once}]),
+    receive
+        {tcp, In_socket, Data} ->
+            Acc_1 = list_to_binary([Acc, Data]),
+            case Acc_1 of
+                <<5, ?NO_AUTH>> when Auth_method == ?NO_AUTH ->
+                    ok = gen_tcp:send(In_socket, <<5, ?NO_AUTH>>),
+                    socks_server_loop(In_socket, Auth_method, <<>>, auth_done);
+                <<5, Num_auth_methods, Auth_methods:Num_auth_methods/binary>> ->
+                    case lists:member(Auth_method, binary_to_list(Auth_methods)) of
+                        true ->
+                            ok = gen_tcp:send(In_socket, <<5, Auth_method>>),
+                            Conn_state = case Auth_method of
+                                             ?NO_AUTH -> auth_done;
+                                             _ -> auth_pending
+                                         end,
+                            socks_server_loop(In_socket, Auth_method, <<>>, Conn_state);
+                        false ->
+                            ok = gen_tcp:send(In_socket, <<5, 16#ff>>),
+                            gen_tcp:close(In_socket)
+                    end;
+                _ ->
+                    ok = gen_tcp:send(In_socket, <<5, 0>>),
+                    gen_tcp:close(In_socket)
+            end;
+        {tcp_closed, In_socket} ->
+            ok;
+        {tcp_error, In_socket, _Rsn} ->
+            ok
+    end;
+socks_server_loop(In_socket, Auth_method, Acc, auth_pending) ->
+    inet:setopts(In_socket, [{active, once}]),
+    receive
+        {tcp, In_socket, Data} ->
+            Acc_1 = list_to_binary([Acc, Data]),
+            case Acc_1 of
+                <<1, U_len, Username:U_len/binary, P_len, Password:P_len/binary>> ->
+                    case check_user_pw(Username, Password) of
+                        ok ->
+                            ok = gen_tcp:send(In_socket, <<1, 0>>),
+                            socks_server_loop(In_socket, Auth_method, <<>>, auth_done);
+                        notok ->
+                            ok = gen_tcp:send(In_socket, <<1, 1>>),
+                            gen_tcp:close(In_socket)
+                    end;
+                _ ->
+                    socks_server_loop(In_socket, Auth_method, Acc_1, auth_pending)
+            end;
+        {tcp_closed, In_socket} ->
+            ok;
+        {tcp_error, In_socket, _Rsn} ->
+            ok
+    end;
+socks_server_loop(In_socket, Auth_method, Acc, auth_done) ->
+    inet:setopts(In_socket, [{active, once}]),
+    receive
+        {tcp, In_socket, Data} ->
+            Acc_1 = list_to_binary([Acc, Data]),
+            case Acc_1 of
+                <<5, 1, 0, Addr_type, Dest_ip:4/binary, Dest_port:16>> when Addr_type == 1->
+                    handle_connect(In_socket, Addr_type, Dest_ip, Dest_port);
+                <<5, 1, 0, Addr_type, Dest_len, Dest_hostname:Dest_len/binary, Dest_port:16>> when Addr_type == 3 ->
+                    handle_connect(In_socket, Addr_type, Dest_hostname, Dest_port);
+                <<5, 1, 0, Addr_type, Dest_ip:16/binary, Dest_port:16>> when Addr_type == 4->
+                    handle_connect(In_socket, Addr_type, Dest_ip, Dest_port);
+                _ ->
+                    socks_server_loop(In_socket, Auth_method, Acc_1, auth_done)
+            end;
+        {tcp_closed, In_socket} ->
+            ok;
+        {tcp_error, In_socket, _Rsn} ->
+            ok
+    end.
+
+handle_connect(In_socket, Addr_type, Dest_host, Dest_port) ->
+    Dest_host_1 = case Addr_type of
+                      1 ->
+                          list_to_tuple(binary_to_list(Dest_host));
+                      3 ->
+                          binary_to_list(Dest_host);
+                      4 ->
+                          list_to_tuple(binary_to_list(Dest_host))
+                  end,
+    case gen_tcp:connect(Dest_host_1, Dest_port, [binary, {active, once}]) of
+        {ok, Out_socket} ->
+            Addr = case Addr_type of
+                       1 ->
+                           <<Dest_host/binary, Dest_port:16>>;
+                       3 ->
+                           Len = size(Dest_host),
+                           <<Len, Dest_host/binary, Dest_port:16>>;
+                       4 ->
+                           <<Dest_host/binary, Dest_port:16>>
+                   end,
+            ok = gen_tcp:send(In_socket, <<5, 0, 0, Addr_type, Addr/binary>>),
+            inet:setopts(In_socket, [{active, once}]),
+            inet:setopts(Out_socket, [{active, once}]),
+            connected_loop(In_socket, Out_socket);
+        _Err ->
+            ok = gen_tcp:send(<<5, 1>>),
+            gen_tcp:close(In_socket)
+    end.
+
+check_user_pw(<<"user">>, <<"password">>) ->
+    ok;
+check_user_pw(_, _) ->
+    notok.
+
+connected_loop(In_socket, Out_socket) ->
+    receive
+        {tcp, In_socket, Data} ->
+            inet:setopts(In_socket, [{active, once}]),
+            ok = gen_tcp:send(Out_socket, Data),
+            connected_loop(In_socket, Out_socket);
+        {tcp, Out_socket, Data} ->
+            inet:setopts(Out_socket, [{active, once}]),
+            ok = gen_tcp:send(In_socket, Data),
+            connected_loop(In_socket, Out_socket);
+        _ ->
+            ok
+    end.

test/ibrowse_test.erl

@@ -33,16 +33,103 @@
          test_303_response_with_no_body/1,
          test_303_response_with_a_body/0,
          test_303_response_with_a_body/1,
+         test_preserve_status_line/0,
          test_binary_headers/0,
          test_binary_headers/1,
          test_generate_body_0/0,
          test_retry_of_requests/0,
          test_retry_of_requests/1,
-	 test_save_to_file_no_content_length/0
+	 test_save_to_file_no_content_length/0,
+         socks5_noauth_test/0,
+         socks5_auth_succ_test/0,
+         socks5_auth_fail_test/0
 	]).
 
 -include_lib("ibrowse/include/ibrowse.hrl").
 
+%%------------------------------------------------------------------------------
+%% Unit Tests
+%%------------------------------------------------------------------------------
+-define(LOCAL_TESTS, [
+                      {local_test_fun, socks5_noauth_test, []},
+                      {local_test_fun, socks5_auth_succ_test, []},
+                      {local_test_fun, socks5_auth_fail_test, []},
+                      {local_test_fun, test_preserve_status_line, []},
+		      {local_test_fun, test_save_to_file_no_content_length, []},
+                      {local_test_fun, test_20122010, []},
+                      {local_test_fun, test_pipeline_head_timeout, []},
+                      {local_test_fun, test_head_transfer_encoding, []},
+                      {local_test_fun, test_head_response_with_body, []},
+                      {local_test_fun, test_303_response_with_a_body, []},
+		      {local_test_fun, test_303_response_with_no_body, []},
+                      {local_test_fun, test_binary_headers, []},
+                      {local_test_fun, test_retry_of_requests, []},
+		      {local_test_fun, verify_chunked_streaming, []},
+		      {local_test_fun, test_chunked_streaming_once, []},
+		      {local_test_fun, test_generate_body_0, []}
+                     ]).
+
+-define(TEST_LIST, [{"http://intranet/messenger", get},
+		    {"http://www.google.co.uk", get},
+		    {"http://www.google.com", get},
+		    {"http://www.google.com", options},
+                    {"https://mail.google.com", get},
+		    {"http://www.sun.com", get},
+		    {"http://www.oracle.com", get},
+		    {"http://www.bbc.co.uk", get},
+		    {"http://www.bbc.co.uk", trace},
+		    {"http://www.bbc.co.uk", options},
+		    {"http://yaws.hyber.org", get},
+		    {"http://jigsaw.w3.org/HTTP/ChunkedScript", get},
+		    {"http://jigsaw.w3.org/HTTP/TE/foo.txt", get},
+		    {"http://jigsaw.w3.org/HTTP/TE/bar.txt", get},
+		    {"http://jigsaw.w3.org/HTTP/connection.html", get},
+		    {"http://jigsaw.w3.org/HTTP/cc.html", get},
+		    {"http://jigsaw.w3.org/HTTP/cc-private.html", get},
+		    {"http://jigsaw.w3.org/HTTP/cc-proxy-revalidate.html", get},
+		    {"http://jigsaw.w3.org/HTTP/cc-nocache.html", get},
+		    {"http://jigsaw.w3.org/HTTP/h-content-md5.html", get},
+		    {"http://jigsaw.w3.org/HTTP/h-retry-after.html", get},
+		    {"http://jigsaw.w3.org/HTTP/h-retry-after-date.html", get},
+		    {"http://jigsaw.w3.org/HTTP/neg", get},
+		    {"http://jigsaw.w3.org/HTTP/negbad", get},
+		    {"http://jigsaw.w3.org/HTTP/400/toolong/", get},
+		    {"http://jigsaw.w3.org/HTTP/300/", get},
+		    {"http://jigsaw.w3.org/HTTP/Basic/", get, [{basic_auth, {"guest", "guest"}}]},
+		    {"http://jigsaw.w3.org/HTTP/CL/", get},
+		    {"http://www.httpwatch.com/httpgallery/chunked/", get},
+                    {"https://github.com", get, [{ssl_options, [{depth, 2}]}]}
+		   ]).
+
+socks5_noauth_test() ->
+    case ibrowse:send_req("http://localhost:8181/success", [], get, [],
+                          [{socks5_host, "localhost"}, {socks5_port, 8282}], 2000) of
+	{ok, "200", _, _} ->
+            success;
+	Err ->
+	    Err
+    end.
+
+socks5_auth_succ_test() ->
+    case ibrowse:send_req("http://localhost:8181/success", [], get, [],
+                          [{socks5_host, "localhost"}, {socks5_port, 8383},
+                           {socks5_user, <<"user">>}, {socks5_password, <<"password">>}], 2000) of
+	{ok, "200", _, _} ->
+            success;
+	Err ->
+	    Err
+    end.
+
+socks5_auth_fail_test() ->
+    case ibrowse:send_req("http://localhost:8181/success", [], get, [],
+                          [{socks5_host, "localhost"}, {socks5_port, 8282},
+                           {socks5_user, <<"user">>}, {socks5_password, <<"wrong_password">>}], 2000) of
+        {error,{conn_failed,{error,unacceptable}}} ->
+            success;
+	Err ->
+	    Err
+    end.
+
 test_stream_once(Url, Method, Options) ->
     test_stream_once(Url, Method, Options, 5000).
 
@@ -212,56 +299,6 @@ dump_errors(Key, Iod) ->
     file:write(Iod, io_lib:format("~p~n", [Term])),
     dump_errors(ets:next(ibrowse_errors, Key), Iod).
 
-%%------------------------------------------------------------------------------
-%% Unit Tests
-%%------------------------------------------------------------------------------
--define(LOCAL_TESTS, [
-                      {local_test_fun, test_20122010, []},
-                      {local_test_fun, test_pipeline_head_timeout, []},
-                      {local_test_fun, test_head_transfer_encoding, []},
-                      {local_test_fun, test_head_response_with_body, []},
-                      {local_test_fun, test_303_response_with_a_body, []},
-		      {local_test_fun, test_303_response_with_no_body, []},
-                      {local_test_fun, test_binary_headers, []},
-                      {local_test_fun, test_retry_of_requests, []},
-		      {local_test_fun, test_save_to_file_no_content_length, []},
-		      {local_test_fun, verify_chunked_streaming, []},
-		      {local_test_fun, test_chunked_streaming_once, []},
-		      {local_test_fun, test_generate_body_0, []}
-                     ]).
-
--define(TEST_LIST, [{"http://intranet/messenger", get},
-		    {"http://www.google.co.uk", get},
-		    {"http://www.google.com", get},
-		    {"http://www.google.com", options},
-                    {"https://mail.google.com", get},
-		    {"http://www.sun.com", get},
-		    {"http://www.oracle.com", get},
-		    {"http://www.bbc.co.uk", get},
-		    {"http://www.bbc.co.uk", trace},
-		    {"http://www.bbc.co.uk", options},
-		    {"http://yaws.hyber.org", get},
-		    {"http://jigsaw.w3.org/HTTP/ChunkedScript", get},
-		    {"http://jigsaw.w3.org/HTTP/TE/foo.txt", get},
-		    {"http://jigsaw.w3.org/HTTP/TE/bar.txt", get},
-		    {"http://jigsaw.w3.org/HTTP/connection.html", get},
-		    {"http://jigsaw.w3.org/HTTP/cc.html", get},
-		    {"http://jigsaw.w3.org/HTTP/cc-private.html", get},
-		    {"http://jigsaw.w3.org/HTTP/cc-proxy-revalidate.html", get},
-		    {"http://jigsaw.w3.org/HTTP/cc-nocache.html", get},
-		    {"http://jigsaw.w3.org/HTTP/h-content-md5.html", get},
-		    {"http://jigsaw.w3.org/HTTP/h-retry-after.html", get},
-		    {"http://jigsaw.w3.org/HTTP/h-retry-after-date.html", get},
-		    {"http://jigsaw.w3.org/HTTP/neg", get},
-		    {"http://jigsaw.w3.org/HTTP/negbad", get},
-		    {"http://jigsaw.w3.org/HTTP/400/toolong/", get},
-		    {"http://jigsaw.w3.org/HTTP/300/", get},
-		    {"http://jigsaw.w3.org/HTTP/Basic/", get, [{basic_auth, {"guest", "guest"}}]},
-		    {"http://jigsaw.w3.org/HTTP/CL/", get},
-		    {"http://www.httpwatch.com/httpgallery/chunked/", get},
-                    {"https://github.com", get, [{ssl_options, [{depth, 2}]}]}
-		   ]).
-
 local_unit_tests() ->
     unit_tests([], ?LOCAL_TESTS).
 
@@ -565,6 +602,16 @@ test_303_response_with_a_body(Url) ->
             {test_failed, Res}
     end.
 
+%% Test that the 'preserve_status_line' option works as expected
+test_preserve_status_line() ->
+    case ibrowse:send_req("http://localhost:8181/ibrowse_preserve_status_line", [], get, [],
+                          [{preserve_status_line, true}]) of
+        {ok, "200", [{ibrowse_status_line,<<"HTTP/1.1 200 OKBlah">>} | _], _} ->
+            success;
+        Res ->
+            {test_failed, Res}
+    end.
+
 %%------------------------------------------------------------------------------
 %% Test that when the save_response_to_file option is used with a server which
 %% does not send the Content-Length header, the response is saved correctly to
@@ -578,7 +625,8 @@ test_save_to_file_no_content_length() ->
 		    lists:flatten(
 		      io_lib:format("test_save_to_file_no_content_length_~p~p~p_~p~p~p.txt", [Y, M, D, H, Mi, S]))]),
     try
-	case ibrowse:send_req("http://localhost:8181/ibrowse_send_file_conn_close", [], get, [], [{save_response_to_file, Test_file}]) of
+	case ibrowse:send_req("http://localhost:8181/ibrowse_send_file_conn_close", [], get, [],
+                              [{save_response_to_file, Test_file}]) of
 	    {ok, "200", _, {file, Test_file}} ->
 		success;
 	    Res ->

test/ibrowse_test_server.erl

@@ -5,6 +5,7 @@
 
 -module(ibrowse_test_server).
 -export([
+         start_server/0,
          start_server/2,
          stop_server/1,
          get_conn_pipeline_depth/0
@@ -16,6 +17,9 @@
 -define(ACCEPT_TIMEOUT_MS, 10000).
 -define(CONN_PIPELINE_DEPTH, conn_pipeline_depth).
 
+start_server() ->
+    start_server(8181, tcp).
+
 start_server(Port, Sock_type) ->
     Fun = fun() ->
 		  Proc_name = server_proc_name(Port),
@@ -42,10 +46,14 @@ start_server(Port, Sock_type) ->
 			  ok
 		  end
 	  end,
-    spawn_link(Fun).
+    spawn_link(Fun),
+    ibrowse_socks_server:start(8282, 0), %% No auth
+    ibrowse_socks_server:start(8383, 2). %% Username/Password auth
 
 stop_server(Port) ->
     catch server_proc_name(Port) ! stop,
+    ibrowse_socks_server:stop(8282),
+    ibrowse_socks_server:stop(8383),
     timer:sleep(2000),  % wait for server to receive msg and unregister
     ok.
 
@@ -246,6 +254,12 @@ process_request(Sock, Sock_type,
                          uri = {abs_path, "/ibrowse_303_with_body_test"}}) ->
     Resp = <<"HTTP/1.1 303 See Other\r\nLocation: http://example.org\r\nContent-Length: 5\r\n\r\nabcde">>,
     do_send(Sock, Sock_type, Resp);
+process_request(Sock, Sock_type,
+                #request{method='GET',
+                         headers = _Headers,
+                         uri = {abs_path, "/ibrowse_preserve_status_line"}}) ->
+    Resp = <<"HTTP/1.1 200 OKBlah\r\nContent-Length: 5\r\n\r\nabcde">>,
+    do_send(Sock, Sock_type, Resp);
 process_request(Sock, Sock_type,
     #request{method='GET',
         headers = _Headers,