Release 2.1.0

Makefile

@@ -9,3 +9,10 @@ clean:
 install: all
 	mkdir -p $(DESTDIR)/lib/ibrowse-$(IBROWSE_VSN)/
 	cp -r ebin $(DESTDIR)/lib/ibrowse-$(IBROWSE_VSN)/
+
+test: all
+	erl -noshell -pa ebin -s ibrowse -s ibrowse_test unit_tests \
+	-s ibrowse_test verify_chunked_streaming \
+	-s ibrowse_test test_chunked_streaming_once \
+	-s erlang halt
+

README

@@ -22,8 +22,83 @@ Version : 2.0.1
 
 Latest version : git://github.com/cmullaparthi/ibrowse.git
 
+CONTRIBUTORS
+============
+The following people have helped maked ibrowse better by reporting bugs,
+supplying patches and also asking for new features. Please write to me if you
+have contributed and I've missed you out.
+
+In alphabetical order:
+
+Adam Kocoloski
+Andrew Tunnell-Jones
+Anthony Molinaro
+Benoit Chesneau
+Chris Newcombe
+Dan Kelley
+Derek Upham
+Eric Merritt
+Erik Reitsma
+Filipe David Manana
+Geoff Cant
+Jeroen Koops
+João Lopes
+Karol Skocik
+Kostis Sagonas
+Matthew Reilly
+Oscar Hellström
+Paul J. Davis
+Peter Kristensen
+Ram Krishnan
+Richard Cameron
+Sean Hinde
+Seth Falcon
+Steve Vinoski
+Thomas Lindgren
+Younès Hafri
+tholschuh (https://github.com/tholschuh/)
+
 CONTRIBUTIONS & CHANGE HISTORY
 ==============================
+25-10-2010 - v2.1.0
+             * Fixed build on OpenSolaris. Bug report and patch from 
+               tholschuh.
+               http://github.com/cmullaparthi/ibrowse/issues/issue/10
+
+             * Fixed behaviour of inactivity_timeout option. Reported by
+               João Lopes.
+               http://github.com/cmullaparthi/ibrowse/issues/issue/11
+
+             * Prevent atom table pollution when bogus URLs are input to 
+               ibrowse. Bug report by João Lopes.
+               http://github.com/cmullaparthi/ibrowse/issues/issue/13
+
+             * Automatically do Chunked-Transfer encoding of request body 
+               when the body is generated by a fun. Patch provided by 
+               Filipe David Manana.
+               http://github.com/cmullaparthi/ibrowse/issues/issue/14
+
+             * Depending on input options, ibrowse sometimes included multiple
+               Content-Length headers. Bug reported by Paul J. Davis
+               http://github.com/cmullaparthi/ibrowse/issues/issue/15
+
+             * Deal with webservers which do not provide a Reason-Phrase on the
+               response Status-Line. Patch provided by Jeroen Koops.
+               http://github.com/cmullaparthi/ibrowse/issues/issue/16
+
+             * Fixed http://github.com/cmullaparthi/ibrowse/issues/issue/17
+               This was reported by Filipe David Manana.
+
+             * Fixed http://github.com/cmullaparthi/ibrowse/issues/issue/19
+               This was reported by Dan Kelley and Filipe David Manana.
+
+             * Added ibrowse:stream_close/1 to close the connection 
+               associated with a certain response stream. Patch provided by
+               João Lopes. 
+
+             * Prevent port number being included in the Host header when port
+               443 is intended. Bug reported by Andrew Tunnell-Jones
+
 24-09-2010 - v2.0.1
              * Removed a spurious io:format statement
 

doc/ibrowse.html

@@ -9,14 +9,14 @@
 <hr>
 
 <h1>Module ibrowse</h1>
-<ul class="index"><li><a href="#description">Description</a></li><li><a href="#index">Function Index</a></li><li><a href="#functions">Function Details</a></li></ul>The ibrowse application implements an HTTP 1.1 client.
+<ul class="index"><li><a href="#description">Description</a></li><li><a href="#index">Function Index</a></li><li><a href="#functions">Function Details</a></li></ul>The ibrowse application implements an HTTP 1.1 client in erlang.
 <p>Copyright © 2005-2010 Chandrashekhar Mullaparthi</p>
 
-<p><b>Version:</b> 2.0.1</p>
+<p><b>Version:</b> 2.1.0</p>
 <p><b>Behaviours:</b> <a href="gen_server.html"><tt>gen_server</tt></a>.</p>
 <p><b>Authors:</b> Chandrashekhar Mullaparthi (<a href="mailto:chandrashekhar dot mullaparthi at gmail dot com"><tt>chandrashekhar dot mullaparthi at gmail dot com</tt></a>).</p>
 
-<h2><a name="description">Description</a></h2><p>The ibrowse application implements an HTTP 1.1 client. This  
+<h2><a name="description">Description</a></h2><p>The ibrowse application implements an HTTP 1.1 client in erlang. This  
 module implements the API of the HTTP client. There is one named  
 process called 'ibrowse' which assists in load balancing and maintaining configuration. There is one load balancing process per unique webserver. There is  
 one process to handle one TCP connection to a webserver  
@@ -87,20 +87,24 @@ send_req/4, send_req/5, send_req/6.
 <tr><td valign="top"><a href="#set_dest-3">set_dest/3</a></td><td>Deprecated.</td></tr>
 <tr><td valign="top"><a href="#set_max_pipeline_size-3">set_max_pipeline_size/3</a></td><td>Set the maximum pipeline size for each connection to a specific Host:Port.</td></tr>
 <tr><td valign="top"><a href="#set_max_sessions-3">set_max_sessions/3</a></td><td>Set the maximum number of connections allowed to a specific Host:Port.</td></tr>
-<tr><td valign="top"><a href="#show_dest_status-0">show_dest_status/0</a></td><td></td></tr>
+<tr><td valign="top"><a href="#show_dest_status-0">show_dest_status/0</a></td><td>Shows some internal information about load balancing.</td></tr>
 <tr><td valign="top"><a href="#show_dest_status-2">show_dest_status/2</a></td><td>Shows some internal information about load balancing to a
   specified Host:Port.</td></tr>
 <tr><td valign="top"><a href="#spawn_link_worker_process-1">spawn_link_worker_process/1</a></td><td>Same as spawn_worker_process/1 except the the calling process
   is linked to the worker process which is spawned.</td></tr>
-<tr><td valign="top"><a href="#spawn_link_worker_process-2">spawn_link_worker_process/2</a></td><td></td></tr>
+<tr><td valign="top"><a href="#spawn_link_worker_process-2">spawn_link_worker_process/2</a></td><td>Same as spawn_worker_process/2 except the the calling process
+  is linked to the worker process which is spawned.</td></tr>
 <tr><td valign="top"><a href="#spawn_worker_process-1">spawn_worker_process/1</a></td><td>Creates a HTTP client process to the specified Host:Port which
   is not part of the load balancing pool.</td></tr>
-<tr><td valign="top"><a href="#spawn_worker_process-2">spawn_worker_process/2</a></td><td></td></tr>
+<tr><td valign="top"><a href="#spawn_worker_process-2">spawn_worker_process/2</a></td><td>Same as spawn_worker_process/1 but takes as input a Host and Port
+  instead of a URL.</td></tr>
 <tr><td valign="top"><a href="#start-0">start/0</a></td><td>Starts the ibrowse process without linking.</td></tr>
 <tr><td valign="top"><a href="#start_link-0">start_link/0</a></td><td>Starts the ibrowse process linked to the calling process.</td></tr>
 <tr><td valign="top"><a href="#stop-0">stop/0</a></td><td>Stop the ibrowse process.</td></tr>
 <tr><td valign="top"><a href="#stop_worker_process-1">stop_worker_process/1</a></td><td>Terminate a worker process spawned using
   spawn_worker_process/2 or spawn_link_worker_process/2.</td></tr>
+<tr><td valign="top"><a href="#stream_close-1">stream_close/1</a></td><td>Tell ibrowse to close the connection associated with the
+  specified stream.</td></tr>
 <tr><td valign="top"><a href="#stream_next-1">stream_next/1</a></td><td>Tell ibrowse to stream the next chunk of data to the
   caller.</td></tr>
 <tr><td valign="top"><a href="#terminate-2">terminate/2</a></td><td></td></tr>
@@ -245,11 +249,15 @@ send_req/4, send_req/5, send_req/6.
   information as it has, such as HTTP Status Code and HTTP Headers. When this happens, the response
   is of the form <code>{error, {Reason, {stat_code, StatusCode}, HTTP_headers}}</code></li>
  
-  <li>The <code>inactivity_timeout</code> option is useful when
-  dealing with large response bodies and/or slow links. In these
-  cases, it might be hard to estimate how long a request will take to
-  complete. In such cases, the client might want to timeout if no
-  data has been received on the link for a certain time interval.</li>
+  <li><p>The <code>inactivity_timeout</code> option is useful when  
+dealing with large response bodies and/or slow links. In these  
+cases, it might be hard to estimate how long a request will take to  
+complete. In such cases, the client might want to timeout if no  
+data has been received on the link for a certain time interval.</p>
+ 
+  This value is also used to close connections which are not in use for
+  the specified timeout value.
+  </li>
  
   <li>
   The <code>connect_timeout</code> option is to specify how long the
@@ -343,7 +351,9 @@ send_req/4, send_req/5, send_req/6.
 <h3 class="function"><a name="show_dest_status-0">show_dest_status/0</a></h3>
 <div class="spec">
 <p><tt>show_dest_status() -&gt; any()</tt></p>
-</div>
+</div><p>Shows some internal information about load balancing. Info
+  about workers spawned using spawn_worker_process/2 or
+  spawn_link_worker_process/2 is not included.</p>
 
 <h3 class="function"><a name="show_dest_status-2">show_dest_status/2</a></h3>
 <div class="spec">
@@ -362,7 +372,8 @@ send_req/4, send_req/5, send_req/6.
 <h3 class="function"><a name="spawn_link_worker_process-2">spawn_link_worker_process/2</a></h3>
 <div class="spec">
 <p><tt>spawn_link_worker_process(Host::string(), Port::integer()) -&gt; {ok, pid()}</tt></p>
-</div>
+</div><p>Same as spawn_worker_process/2 except the the calling process
+  is linked to the worker process which is spawned.</p>
 
 <h3 class="function"><a name="spawn_worker_process-1">spawn_worker_process/1</a></h3>
 <div class="spec">
@@ -383,7 +394,8 @@ send_req/4, send_req/5, send_req/6.
 <h3 class="function"><a name="spawn_worker_process-2">spawn_worker_process/2</a></h3>
 <div class="spec">
 <p><tt>spawn_worker_process(Host::string(), Port::integer()) -&gt; {ok, pid()}</tt></p>
-</div>
+</div><p>Same as spawn_worker_process/1 but takes as input a Host and Port
+  instead of a URL.</p>
 
 <h3 class="function"><a name="start-0">start/0</a></h3>
 <div class="spec">
@@ -407,6 +419,15 @@ send_req/4, send_req/5, send_req/6.
   spawn_worker_process/2 or spawn_link_worker_process/2. Requests in
   progress will get the error response <pre>{error, closing_on_request}</pre></p>
 
+<h3 class="function"><a name="stream_close-1">stream_close/1</a></h3>
+<div class="spec">
+<p><tt>stream_close(Req_id::<a href="#type-req_id">req_id()</a>) -&gt; ok | {error, unknown_req_id}</tt></p>
+</div><p>Tell ibrowse to close the connection associated with the
+  specified stream.  Should be used in conjunction with the
+  <code>stream_to</code> option. Note that all requests in progress on
+  the connection which is serving this Req_id will be aborted, and an
+  error returned.</p>
+
 <h3 class="function"><a name="stream_next-1">stream_next/1</a></h3>
 <div class="spec">
 <p><tt>stream_next(Req_id::<a href="#type-req_id">req_id()</a>) -&gt; ok | {error, unknown_req_id}</tt></p>
@@ -446,6 +467,6 @@ send_req/4, send_req/5, send_req/6.
 <hr>
 
 <div class="navbar"><a name="#navbar_bottom"></a><table width="100%" border="0" cellspacing="0" cellpadding="2" summary="navigation bar"><tr><td><a href="overview-summary.html" target="overviewFrame">Overview</a></td><td><a href="http://www.erlang.org/"><img src="erlang.png" align="right" border="0" alt="erlang logo"></a></td></tr></table></div>
-<p><i>Generated by EDoc, Sep 24 2010, 06:42:36.</i></p>
+<p><i>Generated by EDoc, Nov 10 2010, 06:04:33.</i></p>
 </body>
 </html>

ebin/ibrowse.app

@@ -1,6 +1,6 @@
 {application, ibrowse,
         [{description, "HTTP client application"},
-         {vsn, "2.0.0"},
+         {vsn, "2.1.0"},
          {modules, [ ibrowse, 
 		     ibrowse_http_client, 
 		     ibrowse_app, 

src/Makefile

@@ -24,7 +24,7 @@ $(EBIN)/%.beam: %.erl
 	${ERLC} $(COMPILER_OPTIONS) $(INCLUDE_DIRS) -o ../ebin $<
 
 $(EBIN)/%.app:	%.app.src ../vsn.mk Makefile
-	sed -e s^%IBROWSE_VSN%^$(IBROWSE_VSN)^ \
+	sed -e s/%IBROWSE_VSN%/$(IBROWSE_VSN)/ \
 		 $< > $@
 
 clean:

src/ibrowse.erl

@@ -7,8 +7,8 @@
 %%%-------------------------------------------------------------------
 %% @author Chandrashekhar Mullaparthi <chandrashekhar dot mullaparthi at gmail dot com>
 %% @copyright 2005-2010 Chandrashekhar Mullaparthi
-%% @version 2.0.1
-%% @doc The ibrowse application implements an HTTP 1.1 client. This
+%% @version 2.1.0
+%% @doc The ibrowse application implements an HTTP 1.1 client in erlang. This
 %% module implements the API of the HTTP client. There is one named
 %% process called 'ibrowse' which assists in load balancing and maintaining configuration. There is one load balancing process per unique webserver. There is
 %% one process to handle one TCP connection to a webserver
@@ -87,6 +87,7 @@
          send_req_direct/6,
          send_req_direct/7,
          stream_next/1,
+         stream_close/1,
          set_max_sessions/3,
          set_max_pipeline_size/3,
          set_dest/3,
@@ -201,7 +202,11 @@ send_req(Url, Headers, Method, Body) ->
 %% dealing with large response bodies and/or slow links. In these
 %% cases, it might be hard to estimate how long a request will take to
 %% complete. In such cases, the client might want to timeout if no
-%% data has been received on the link for a certain time interval.</li>
+%% data has been received on the link for a certain time interval.
+%% 
+%% This value is also used to close connections which are not in use for 
+%% the specified timeout value.
+%% </li>
 %%
 %% <li>
 %% The <code>connect_timeout</code> option is to specify how long the
@@ -458,6 +463,8 @@ ensure_bin({Fun, _} = Body) when is_function(Fun) -> Body.
 spawn_worker_process(Url) ->
     ibrowse_http_client:start(Url).
 
+%% @doc Same as spawn_worker_process/1 but takes as input a Host and Port
+%% instead of a URL.
 %% @spec spawn_worker_process(Host::string(), Port::integer()) -> {ok, pid()}
 spawn_worker_process(Host, Port) ->
     ibrowse_http_client:start({Host, Port}).
@@ -468,6 +475,8 @@ spawn_worker_process(Host, Port) ->
 spawn_link_worker_process(Url) ->
     ibrowse_http_client:start_link(Url).
 
+%% @doc Same as spawn_worker_process/2 except the the calling process
+%% is linked to the worker process which is spawned.
 %% @spec spawn_link_worker_process(Host::string(), Port::integer()) -> {ok, pid()}
 spawn_link_worker_process(Host, Port) ->
     ibrowse_http_client:start_link({Host, Port}).
@@ -524,6 +533,21 @@ stream_next(Req_id) ->
             ok
     end.
 
+%% @doc Tell ibrowse to close the connection associated with the
+%% specified stream.  Should be used in conjunction with the
+%% <code>stream_to</code> option. Note that all requests in progress on
+%% the connection which is serving this Req_id will be aborted, and an
+%% error returned.
+%% @spec stream_close(Req_id :: req_id()) -> ok | {error, unknown_req_id}
+stream_close(Req_id) ->    
+    case ets:lookup(ibrowse_stream, {req_id_pid, Req_id}) of
+        [] ->
+            {error, unknown_req_id};
+        [{_, Pid}] ->
+            catch Pid ! {stream_close, Req_id},
+            ok
+    end.
+
 %% @doc Turn tracing on for the ibrowse process
 trace_on() ->
     ibrowse ! {trace, true}.
@@ -553,6 +577,9 @@ all_trace_off() ->
     ibrowse ! all_trace_off,
     ok.
 
+%% @doc Shows some internal information about load balancing. Info
+%% about workers spawned using spawn_worker_process/2 or
+%% spawn_link_worker_process/2 is not included.
 show_dest_status() ->
     Dests = lists:filter(fun({lb_pid, {Host, Port}, _}) when is_list(Host),
                                                              is_integer(Port) ->

src/ibrowse_http_client.erl

@@ -37,6 +37,7 @@
 -include("ibrowse.hrl").
 
 -record(state, {host, port, connect_timeout,
+                inactivity_timer_ref,
                 use_proxy = false, proxy_auth_digest,
                 ssl_options = [], is_ssl = false, socket,
                 proxy_tunnel_setup = false,
@@ -192,6 +193,12 @@ handle_info({stream_next, Req_id}, #state{socket = Socket,
 handle_info({stream_next, _Req_id}, State) ->
     {noreply, State};
 
+handle_info({stream_close, _Req_id}, State) ->
+    shutting_down(State),
+    do_close(State),
+    do_error_reply(State, closing_on_request),
+    {stop, normal, ok, State};
+
 handle_info({tcp_closed, _Sock}, State) ->    
     do_trace("TCP connection closed by peer!~n", []),
     handle_sock_closed(State),
@@ -221,6 +228,7 @@ handle_info({req_timedout, From}, State) ->
     end;
 
 handle_info(timeout, State) ->
+    do_trace("Inactivity timeout triggered. Shutting down connection~n", []),
     shutting_down(State),
     do_error_reply(State, req_timedout),
     {stop, normal, State};
@@ -273,8 +281,8 @@ handle_sock_data(Data, #state{status = get_header}=State) ->
             {stop, normal, State};
         State_1 ->
             active_once(State_1),
-            set_inac_timer(State_1),
-            {noreply, State_1}
+            State_2 = set_inac_timer(State_1),
+            {noreply, State_2}
     end;
 
 handle_sock_data(Data, #state{status           = get_body,
@@ -293,8 +301,8 @@ handle_sock_data(Data, #state{status           = get_body,
                     {stop, normal, State};
                 State_1 ->
                     active_once(State_1),
-                    set_inac_timer(State_1),
-                    {noreply, State_1}
+                    State_2 = set_inac_timer(State_1),
+                    {noreply, State_2}
             end;
         _ ->
             case parse_11_response(Data, State) of
@@ -314,12 +322,12 @@ handle_sock_data(Data, #state{status           = get_body,
                             active_once(State_1)
                     end,
                     State_2 = State_1#state{interim_reply_sent = false},
-                    set_inac_timer(State_2),
-                    {noreply, State_2};
+                    State_3 = set_inac_timer(State_2),
+                    {noreply, State_3};
                 State_1 ->
                     active_once(State_1),
-                    set_inac_timer(State_1),
-                    {noreply, State_1}
+                    State_2 = set_inac_timer(State_1),
+                    {noreply, State_2}
             end
     end.
 
@@ -507,29 +515,37 @@ do_send(Req, #state{socket = Sock, is_ssl = false}) ->  gen_tcp:send(Sock, Req).
 %%                       {fun_arity_0}         |
 %%                       {fun_arity_1, term()}
 %% error() = term()
-do_send_body(Source, State) when is_function(Source) ->
-    do_send_body({Source}, State);
-do_send_body({Source}, State) when is_function(Source) ->
-    do_send_body1(Source, Source(), State);
-do_send_body({Source, Source_state}, State) when is_function(Source) ->
-    do_send_body1(Source, Source(Source_state), State);
-do_send_body(Body, State) ->
+do_send_body(Source, State, TE) when is_function(Source) ->
+    do_send_body({Source}, State, TE);
+do_send_body({Source}, State, TE) when is_function(Source) ->
+    do_send_body1(Source, Source(), State, TE);
+do_send_body({Source, Source_state}, State, TE) when is_function(Source) ->
+    do_send_body1(Source, Source(Source_state), State, TE);
+do_send_body(Body, State, _TE) ->
     do_send(Body, State).
 
-do_send_body1(Source, Resp, State) ->
+do_send_body1(Source, Resp, State, TE) ->
     case Resp of
         {ok, Data} ->
-            do_send(Data, State),
-            do_send_body({Source}, State);
+            do_send(maybe_chunked_encode(Data, TE), State),
+            do_send_body({Source}, State, TE);
         {ok, Data, New_source_state} ->
-            do_send(Data, State),
-            do_send_body({Source, New_source_state}, State);
+            do_send(maybe_chunked_encode(Data, TE), State),
+            do_send_body({Source, New_source_state}, State, TE);
+        eof when TE == true ->
+            do_send(<<"0\r\n\r\n">>, State),
+            ok;
         eof ->
             ok;
         Err ->
             Err
     end.
 
+maybe_chunked_encode(Data, false) ->
+    Data;
+maybe_chunked_encode(Data, true) ->
+    [ibrowse_lib:dec2hex(4, size(to_binary(Data))), "\r\n", Data, "\r\n"].
+
 do_close(#state{socket = undefined})            ->  ok;
 do_close(#state{socket = Sock,
                 is_ssl = true,
@@ -619,11 +635,13 @@ send_req_1(From,
     {Req, Body_1} = make_request(connect, Pxy_auth_headers,
                                  Path, Path,
                                  [], Options, State_1),
+    TE = is_chunked_encoding_specified(Options),
     trace_request(Req),
     case do_send(Req, State) of
         ok ->
-            case do_send_body(Body_1, State_1) of
+            case do_send_body(Body_1, State_1, TE) of
                 ok ->
+                    trace_request_body(Body_1),
                     active_once(State_1),
                     Ref = case Timeout of
                               infinity ->
@@ -636,8 +654,8 @@ send_req_1(From,
                                             send_timer = Ref,
                                             proxy_tunnel_setup = in_progress,
                                             tunnel_setup_queue = [{From, Url, Headers, Method, Body, Options, Timeout}]},
-                    set_inac_timer(State_1),
-                    {noreply, State_2};
+                    State_3 = set_inac_timer(State_2),
+                    {noreply, State_3};
                 Err ->
                     shutting_down(State_1),
                     do_trace("Send failed... Reason: ~p~n", [Err]),
@@ -706,10 +724,12 @@ send_req_1(From,
                                  AbsPath, RelPath, Body, Options, State_1),
     trace_request(Req),
     do_setopts(Socket, Caller_socket_options, Is_ssl),
+    TE = is_chunked_encoding_specified(Options),
     case do_send(Req, State_1) of
         ok ->
-            case do_send_body(Body_1, State_1) of
+            case do_send_body(Body_1, State_1, TE) of
                 ok ->
+                    trace_request_body(Body_1),
                     State_2 = inc_pipeline_counter(State_1),
                     active_once(State_2),
                     Ref = case Timeout of
@@ -732,8 +752,8 @@ send_req_1(From,
                         _ ->
                             gen_server:reply(From, {ibrowse_req_id, ReqId})
                     end,
-                    set_inac_timer(State_1),
-                    {noreply, State_3};
+                    State_4 = set_inac_timer(State_3),
+                    {noreply, State_4};
                 Err ->
                     shutting_down(State_1),
                     do_trace("Send failed... Reason: ~p~n", [Err]),
@@ -759,6 +779,7 @@ maybe_modify_headers(#url{host = Host, port = Port} = Url,
                                   false ->
                                       case Port of
                                           80 -> Host;
+                                          443 -> Host;
                                           _ -> [Host, ":", integer_to_list(Port)]
                                       end;
                                   {value, {_, Host_h_val}} ->
@@ -802,31 +823,42 @@ http_auth_digest(Username, Password) ->
 make_request(Method, Headers, AbsPath, RelPath, Body, Options,
              #state{use_proxy = UseProxy, is_ssl = Is_ssl}) ->
     HttpVsn = http_vsn_string(get_value(http_vsn, Options, {1,1})),
+    Fun1 = fun({X, Y}) when is_atom(X) ->
+                   {to_lower(atom_to_list(X)), X, Y};
+              ({X, Y}) when is_list(X) ->
+                   {to_lower(X), X, Y}
+           end,
+    Headers_0 = [Fun1(X) || X <- Headers],
     Headers_1 =
-        case get_value(content_length, Headers, false) of
-            false when (Body == []) or
-            (Body == <<>>) or
-            is_tuple(Body) or
-            is_function(Body) ->
-                Headers;
+        case lists:keysearch("content-length", 1, Headers_0) of
+            false when (Body == []) orelse
+                       (Body == <<>>) orelse
+                       is_tuple(Body) orelse
+                       is_function(Body) ->
+                Headers_0;
             false when is_binary(Body) ->
-                [{"content-length", integer_to_list(size(Body))} | Headers];
-            false ->
-                [{"content-length", integer_to_list(length(Body))} | Headers];
+                [{"content-length", "content-length", integer_to_list(size(Body))} | Headers_0];
+            false when is_list(Body) ->
+                [{"content-length", "content-length", integer_to_list(length(Body))} | Headers_0];
             _ ->
-                Headers
+                %% Content-Length is already specified
+                Headers_0
         end,
     {Headers_2, Body_1} =
-        case get_value(transfer_encoding, Options, false) of
+        case is_chunked_encoding_specified(Options) of
             false ->
-                {Headers_1, Body};
-            {chunked, ChunkSize} ->
-                {[{X, Y} || {X, Y} <- Headers_1,
-                            X /= "Content-Length",
-                            X /= "content-length",
-                            X /= content_length] ++
+                {[{Y, Z} || {_, Y, Z} <- Headers_1], Body};
+            true ->
+                Chunk_size_1 = case get_value(transfer_encoding, Options) of
+                                  chunked ->
+                                      5120;
+                                  {chunked, Chunk_size} ->
+                                      Chunk_size
+                              end,
+                {[{Y, Z} || {X, Y, Z} <- Headers_1,
+                            X /= "content-length"] ++
                  [{"Transfer-Encoding", "chunked"}],
-                 chunk_request_body(Body, ChunkSize)}
+                 chunk_request_body(Body, Chunk_size_1)}
         end,
     Headers_3 = cons_headers(Headers_2),
     Uri = case get_value(use_absolute_uri, Options, false) or UseProxy of
@@ -842,6 +874,16 @@ make_request(Method, Headers, AbsPath, RelPath, Body, Options,
           end,
     {[method(Method), " ", Uri, " ", HttpVsn, crnl(), Headers_3, crnl()], Body_1}.
 
+is_chunked_encoding_specified(Options) ->
+    case get_value(transfer_encoding, Options, false) of
+        false ->
+            false;
+        {chunked, _} -> 
+            true;
+        chunked ->
+            true
+    end.
+
 http_vsn_string({0,9}) -> "HTTP/0.9";
 http_vsn_string({1,0}) -> "HTTP/1.0";
 http_vsn_string({1,1}) -> "HTTP/1.1".
@@ -873,6 +915,9 @@ encode_headers([{Name,Val} | T], Acc) when is_atom(Name) ->
 encode_headers([], Acc) ->
     lists:reverse(Acc).
 
+chunk_request_body(Body, _ChunkSize) when is_tuple(Body) orelse
+                                          is_function(Body) ->
+    Body;
 chunk_request_body(Body, ChunkSize) ->
     chunk_request_body(Body, ChunkSize, []).
 
@@ -1060,7 +1105,7 @@ upgrade_to_ssl(#state{socket = Socket,
 
 send_queued_requests([], State) ->
     do_trace("Sent all queued requests via SSL connection~n", []),
-    State#state{tunnel_setup_queue = done};
+    State#state{tunnel_setup_queue = []};
 send_queued_requests([{From, Url, Headers, Method, Body, Options, Timeout} | Q],
                      State) ->
     case send_req_1(From, Url, Headers, Method, Body, Options, Timeout, State) of
@@ -1217,7 +1262,6 @@ handle_response(#request{from=From, stream_to=StreamTo, req_id=ReqId,
                        reply_buffer  = RepBuf,
                        recvd_headers = RespHeaders}=State) when SaveResponseToFile /= false ->
     Body = RepBuf,
-    State_1 = set_cur_request(State),
     file:close(Fd),
     ResponseBody = case TmpFilename of
                        undefined ->
@@ -1232,9 +1276,9 @@ handle_response(#request{from=From, stream_to=StreamTo, req_id=ReqId,
                 false ->
                     {ok, SCode, Resp_headers_1, ResponseBody}
             end,
-    State_2 = do_reply(State_1, From, StreamTo, ReqId, Resp_format, Reply),
+    State_1 = do_reply(State, From, StreamTo, ReqId, Resp_format, Reply),
     cancel_timer(ReqTimer, {eat_message, {req_timedout, From}}),
-    State_2;
+    set_cur_request(State_1);
 handle_response(#request{from=From, stream_to=StreamTo, req_id=ReqId,
                          response_format = Resp_format,
                          options = Options},
@@ -1245,7 +1289,6 @@ handle_response(#request{from=From, stream_to=StreamTo, req_id=ReqId,
                        reply_buffer     = RepBuf,
                        send_timer       = ReqTimer} = State) ->
     Body = RepBuf,
-%%    State_1 = set_cur_request(State),
     {Resp_headers_1, Raw_headers_1} = maybe_add_custom_headers(Resp_headers, Raw_headers, Options),
     Reply = case get_value(give_raw_headers, Options, false) of
                 true ->
@@ -1253,15 +1296,8 @@ handle_response(#request{from=From, stream_to=StreamTo, req_id=ReqId,
                 false ->
                     {ok, SCode, Resp_headers_1, Body}
             end,
-    State_1 = case get(conn_close) of
-        "close" ->
-            do_reply(State, From, StreamTo, ReqId, Resp_format, Reply),
-            exit(normal);
-        _ ->
-            State_1_1 = do_reply(State, From, StreamTo, ReqId, Resp_format, Reply),
-            cancel_timer(ReqTimer, {eat_message, {req_timedout, From}}),
-            State_1_1
-    end,
+    State_1 = do_reply(State, From, StreamTo, ReqId, Resp_format, Reply),
+    cancel_timer(ReqTimer, {eat_message, {req_timedout, From}}),
     set_cur_request(State_1).
 
 reset_state(State) ->
@@ -1353,6 +1389,8 @@ parse_status_line([32 | T], get_prot_vsn, ProtVsn, StatCode) ->
     parse_status_line(T, get_status_code, ProtVsn, StatCode);
 parse_status_line([32 | T], get_status_code, ProtVsn, StatCode) ->
     {ok, lists:reverse(ProtVsn), lists:reverse(StatCode), T};
+parse_status_line([], get_status_code, ProtVsn, StatCode) ->
+    {ok, lists:reverse(ProtVsn), lists:reverse(StatCode), []};
 parse_status_line([H | T], get_prot_vsn, ProtVsn, StatCode) ->
     parse_status_line(T, get_prot_vsn, [H|ProtVsn], StatCode);
 parse_status_line([H | T], get_status_code, ProtVsn, StatCode) ->
@@ -1710,28 +1748,61 @@ get_stream_chunk_size(Options) ->
     end.
 
 set_inac_timer(State) ->
-    set_inac_timer(State, get_inac_timeout(State)).
+    cancel_timer(State#state.inactivity_timer_ref),
+    set_inac_timer(State#state{inactivity_timer_ref = undefined},
+                   get_inac_timeout(State)).
 
-set_inac_timer(_State, Timeout) when is_integer(Timeout) ->
-    erlang:send_after(Timeout, self(), timeout);
-set_inac_timer(_, _) ->
-    undefined.
+set_inac_timer(State, Timeout) when is_integer(Timeout) ->
+    Ref = erlang:send_after(Timeout, self(), timeout),
+    State#state{inactivity_timer_ref = Ref};
+set_inac_timer(State, _) ->
+    State.
 
 get_inac_timeout(#state{cur_req = #request{options = Opts}}) -> 
     get_value(inactivity_timeout, Opts, infinity);
 get_inac_timeout(#state{cur_req = undefined}) ->
-    infinity.
+    case ibrowse:get_config_value(inactivity_timeout, undefined) of
+        Val when is_integer(Val) ->
+            Val;
+        _ ->
+            case application:get_env(ibrowse, inactivity_timeout) of
+                {ok, Val} when is_integer(Val), Val > 0 ->
+                    Val;
+                _ ->
+                    10000
+            end
+    end.
 
 trace_request(Req) ->
     case get(my_trace_flag) of
         true ->
             %%Avoid the binary operations if trace is not on...
-            NReq = binary_to_list(list_to_binary(Req)),
+            NReq = to_binary(Req),
             do_trace("Sending request: ~n"
                      "--- Request Begin ---~n~s~n"
                      "--- Request End ---~n", [NReq]);
         _ -> ok
     end.
 
+trace_request_body(Body) ->
+    case get(my_trace_flag) of
+        true ->
+            %%Avoid the binary operations if trace is not on...
+            NBody = to_binary(Body),
+            case size(NBody) > 1024 of
+                true ->
+                    ok;
+                false ->
+                    do_trace("Sending request body: ~n"
+                             "--- Request Body Begin ---~n~s~n"
+                             "--- Request Body End ---~n", [NBody])
+            end;
+        false ->
+            ok
+    end.
+
 to_integer(X) when is_list(X)    -> list_to_integer(X); 
 to_integer(X) when is_integer(X) -> X.
+
+to_binary(X) when is_list(X)   -> list_to_binary(X); 
+to_binary(X) when is_binary(X) -> X.

src/ibrowse_lib.erl

@@ -208,7 +208,7 @@ parse_url(Url) ->
 parse_url([$:, $/, $/ | _], get_protocol, Url, []) ->
     {invalid_uri_1, Url};
 parse_url([$:, $/, $/ | T], get_protocol, Url, TmpAcc) ->
-    Prot = list_to_atom(lists:reverse(TmpAcc)),
+    Prot = list_to_existing_atom(lists:reverse(TmpAcc)),
     parse_url(T, get_username, 
               Url#url{protocol = Prot},
               []);

src/ibrowse_test.erl

@@ -217,14 +217,18 @@ dump_errors(Key, Iod) ->
 		    {"http://jigsaw.w3.org/HTTP/300/", get},
 		    {"http://jigsaw.w3.org/HTTP/Basic/", get, [{basic_auth, {"guest", "guest"}}]},
 		    {"http://jigsaw.w3.org/HTTP/CL/", get},
-		    {"http://www.httpwatch.com/httpgallery/chunked/", get}
+		    {"http://www.httpwatch.com/httpgallery/chunked/", get},
+                    {"https://github.com", get, [{ssl_options, [{depth, 2}]}]}
 		   ]).
 
 unit_tests() ->
     unit_tests([]).
 
 unit_tests(Options) ->
+    application:start(crypto),
+    application:start(public_key),
     application:start(ssl),
+    ibrowse:start(),
     Options_1 = Options ++ [{connect_timeout, 5000}],
     {Pid, Ref} = erlang:spawn_monitor(?MODULE, unit_tests_1, [self(), Options_1]),
     receive 
@@ -249,32 +253,45 @@ verify_chunked_streaming() ->
     verify_chunked_streaming([]).
 
 verify_chunked_streaming(Options) ->
+    io:format("~nVerifying that chunked streaming is working...~n", []),
     Url = "http://www.httpwatch.com/httpgallery/chunked/",
-    io:format("URL: ~s~n", [Url]),
-    io:format("Fetching data without streaming...~n", []),
+    io:format("  URL: ~s~n", [Url]),
+    io:format("  Fetching data without streaming...~n", []),
     Result_without_streaming = ibrowse:send_req(
 				 Url, [], get, [],
 				 [{response_format, binary} | Options]),
-    io:format("Fetching data with streaming as list...~n", []),
+    io:format("  Fetching data with streaming as list...~n", []),
     Async_response_list = do_async_req_list(
 			    Url, get, [{response_format, list} | Options]),
-    io:format("Fetching data with streaming as binary...~n", []),
+    io:format("  Fetching data with streaming as binary...~n", []),
     Async_response_bin = do_async_req_list(
 			   Url, get, [{response_format, binary} | Options]),
-    io:format("Fetching data with streaming as binary, {active, once}...~n", []),
+    io:format("  Fetching data with streaming as binary, {active, once}...~n", []),
     Async_response_bin_once = do_async_req_list(
                                 Url, get, [once, {response_format, binary} | Options]),
-    compare_responses(Result_without_streaming, Async_response_list, Async_response_bin),
-    compare_responses(Result_without_streaming, Async_response_list, Async_response_bin_once).
+    Res1 = compare_responses(Result_without_streaming, Async_response_list, Async_response_bin),
+    Res2 = compare_responses(Result_without_streaming, Async_response_list, Async_response_bin_once),
+    case {Res1, Res2} of
+        {success, success} ->
+            io:format("  Chunked streaming working~n", []);
+        _ ->
+            ok
+    end.
 
 test_chunked_streaming_once() ->
     test_chunked_streaming_once([]).
 
 test_chunked_streaming_once(Options) ->
+    io:format("~nTesting chunked streaming with the {stream_to, {Pid, once}} option...~n", []),
     Url = "http://www.httpwatch.com/httpgallery/chunked/",
-    io:format("URL: ~s~n", [Url]),
-    io:format("Fetching data with streaming as binary, {active, once}...~n", []),
-    do_async_req_list(Url, get, [once, {response_format, binary} | Options]).
+    io:format("  URL: ~s~n", [Url]),
+    io:format("  Fetching data with streaming as binary, {active, once}...~n", []),
+    case do_async_req_list(Url, get, [once, {response_format, binary} | Options]) of
+        {ok, _, _, _} ->
+            io:format("  Success!~n", []);
+        Err ->
+            io:format("  Fail: ~p~n", [Err])
+    end.
 
 compare_responses({ok, St_code, _, Body}, {ok, St_code, _, Body}, {ok, St_code, _, Body}) ->
     success;
@@ -310,7 +327,7 @@ do_async_req_list(Url, Method, Options) ->
     {Pid,_} = erlang:spawn_monitor(?MODULE, i_do_async_req_list,
 				   [self(), Url, Method, 
 				    Options ++ [{stream_chunk_size, 1000}]]),
-    io:format("Spawned process ~p~n", [Pid]),
+%%    io:format("Spawned process ~p~n", [Pid]),
     wait_for_resp(Pid).
 
 wait_for_resp(Pid) ->
@@ -354,7 +371,7 @@ wait_for_async_resp(Req_id, Options, Acc_Stat_code, Acc_Headers, Body) ->
             maybe_stream_next(Req_id, Options),
 	    wait_for_async_resp(Req_id, Options, StatCode, Headers, Body);
 	{ibrowse_async_response_end, Req_id} ->
-            io:format("Recvd end of response.~n", []),
+            %% io:format("Recvd end of response.~n", []),
 	    Body_1 = list_to_binary(lists:reverse(Body)),
 	    {ok, Acc_Stat_code, Acc_Headers, Body_1};
 	{ibrowse_async_response, Req_id, Data} ->
@@ -384,7 +401,7 @@ execute_req(Url, Method, Options) ->
 	{ok, SCode, _H, _B} ->
 	    io:format("Status code: ~p~n", [SCode]);
 	Err ->
-	    io:format("Err -> ~p~n", [Err])
+	    io:format("~p~n", [Err])
     end.
 
 drv_ue_test() ->

test/ibrowse_test_server.erl

@@ -0,0 +1,110 @@
+%%% File    : ibrowse_test_server.erl
+%%% Author  : Chandrashekhar Mullaparthi <chandrashekhar.mullaparthi@t-mobile.co.uk>
+%%% Description : A server to simulate various test scenarios
+%%% Created : 17 Oct 2010 by Chandrashekhar Mullaparthi <chandrashekhar.mullaparthi@t-mobile.co.uk>
+
+-module(ibrowse_test_server).
+-export([
+         start_server/2,
+         stop_server/1
+        ]).
+
+-record(request, {method, uri, version, headers = [], body = []}).
+
+start_server(Port, Sock_type) ->
+    Fun = fun() ->
+                  register(server_proc_name(Port), self()),
+                  case do_listen(Sock_type, Port, [{active, false},
+                                                   {packet, http}]) of
+                      {ok, Sock} ->
+                          do_trace("Server listening on port: ~p~n", [Port]),
+                          accept_loop(Sock, Sock_type);
+                      Err ->
+                          do_trace("Failed to start server on port ~p. ~p~n",
+                                   [Port, Err]),
+                          Err
+                  end
+          end,
+    spawn(Fun).
+
+stop_server(Port) ->
+    exit(whereis(server_proc_name(Port)), kill).
+
+server_proc_name(Port) ->
+    list_to_atom("ibrowse_test_server_"++integer_to_list(Port)).
+
+do_listen(tcp, Port, Opts) ->
+    gen_tcp:listen(Port, Opts);
+do_listen(ssl, Port, Opts) ->
+    application:start(crypto),
+    application:start(ssl),
+    ssl:listen(Port, Opts).
+
+do_accept(tcp, Listen_sock) ->
+    gen_tcp:accept(Listen_sock);
+do_accept(ssl, Listen_sock) ->
+    ssl:ssl_accept(Listen_sock).
+
+accept_loop(Sock, Sock_type) ->
+    case do_accept(Sock_type, Sock) of
+        {ok, Conn} ->
+            Pid = spawn_link(
+              fun() ->
+                      server_loop(Conn, Sock_type, #request{})
+              end),
+            set_controlling_process(Conn, Sock_type, Pid),
+            Pid ! {setopts, [{active, true}]},
+            accept_loop(Sock, Sock_type);
+        Err ->
+            Err
+    end.
+
+set_controlling_process(Sock, tcp, Pid) ->
+    gen_tcp:controlling_process(Sock, Pid);
+set_controlling_process(Sock, ssl, Pid) ->
+    ssl:controlling_process(Sock, Pid).
+
+setopts(Sock, tcp, Opts) ->
+    inet:setopts(Sock, Opts);
+setopts(Sock, ssl, Opts) ->
+    ssl:setopts(Sock, Opts).
+
+server_loop(Sock, Sock_type, #request{headers = Headers} = Req) ->
+    receive
+        {http, Sock, {http_request, HttpMethod, HttpUri, HttpVersion}} ->
+            server_loop(Sock, Sock_type, Req#request{method = HttpMethod,
+                                                     uri = HttpUri,
+                                                     version = HttpVersion});
+        {http, Sock, {http_header, _, _, _, _} = H} ->
+            server_loop(Sock, Sock_type, Req#request{headers = [H | Headers]});
+        {http, Sock, http_eoh} ->
+            process_request(Sock, Sock_type, Req),
+            server_loop(Sock, Sock_type, #request{});
+        {http, Sock, {http_error, Err}} ->
+            do_trace("Error parsing HTTP request:~n"
+                     "Req so far : ~p~n"
+                     "Err        : ", [Req, Err]),
+            exit({http_error, Err});
+        {setopts, Opts} ->
+            setopts(Sock, Sock_type, Opts),
+            server_loop(Sock, Sock_type, Req);
+        Other ->
+            do_trace("Recvd unknown msg: ~p~n", [Other]),
+            exit({unknown_msg, Other})
+    after 5000 ->
+            do_trace("Timing out client connection~n", []),
+            ok
+    end.
+
+do_trace(Fmt, Args) ->
+    io:format("~s -- " ++ Fmt, [ibrowse_lib:printable_date() | Args]).
+
+process_request(Sock, Sock_type, Req) ->
+    do_trace("Recvd req: ~p~n", [Req]),
+    Resp = <<"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n">>,
+    do_send(Sock, Sock_type, Resp).
+
+do_send(Sock, tcp, Resp) ->
+    ok = gen_tcp:send(Sock, Resp);
+do_send(Sock, ssl, Resp) ->
+    ok = ssl:send(Sock, Resp).

vsn.mk

@@ -1,2 +1,2 @@
-IBROWSE_VSN = 2.0.1
+IBROWSE_VSN = 2.1.0