29-07-2009 - * The ETS table created for load balancing of requests was not

being deleted which led to the node not being able to create
               any more ETS tables if queries were made to many number of
               webservers. ibrowse now deletes the ETS table it creates once the
               last connection to a webserver is dropped.
               Reported by Seth Falcon.
             * Spurious data being returned at end of body in certain cases of
               chunked encoded responses from the server.
               Reported by Chris Newcombe.

README

@@ -18,12 +18,22 @@ ibrowse is available under two different licenses. LGPL and the BSD license.
 
 Comments to : Chandrashekhar.Mullaparthi@gmail.com
 
-Version : 1.5.1
+Version : 1.5.2
 
 Latest version : git://github.com/cmullaparthi/ibrowse.git
 
 CONTRIBUTIONS & CHANGE HISTORY
 ==============================
+29-07-2009 - * The ETS table created for load balancing of requests was not
+               being deleted which led to the node not being able to create
+               any more ETS tables if queries were made to many number of 
+               webservers. ibrowse now deletes the ETS table it creates once the
+               last connection to a webserver is dropped. 
+               Reported by Seth Falcon.
+             * Spurious data being returned at end of body in certain cases of
+               chunked encoded responses from the server. 
+               Reported by Chris Newcombe.
+
 03-07-2009 - Added option {stream_to, {Pid, once}} which allows the caller
              to control when it wants to receive more data. If this option
              is used, the call ibrowse:stream_next(Req_id) should be used

doc/ibrowse.html

@@ -12,7 +12,7 @@
 <ul class="index"><li><a href="#description">Description</a></li><li><a href="#index">Function Index</a></li><li><a href="#functions">Function Details</a></li></ul>The ibrowse application implements an HTTP 1.1 client.
 <p>Copyright © 2005-2009 Chandrashekhar Mullaparthi</p>
 
-<p><b>Version:</b> 1.5.1</p>
+<p><b>Version:</b> 1.5.2</p>
 <p><b>Behaviours:</b> <a href="gen_server.html"><tt>gen_server</tt></a>.</p>
 <p><b>Authors:</b> Chandrashekhar Mullaparthi (<a href="mailto:chandrashekhar dot mullaparthi at gmail dot com"><tt>chandrashekhar dot mullaparthi at gmail dot com</tt></a>).</p>
 
@@ -90,6 +90,7 @@ send_req/4, send_req/5, send_req/6.
 <tr><td valign="top"><a href="#set_dest-3">set_dest/3</a></td><td>Deprecated.</td></tr>
 <tr><td valign="top"><a href="#set_max_pipeline_size-3">set_max_pipeline_size/3</a></td><td>Set the maximum pipeline size for each connection to a specific Host:Port.</td></tr>
 <tr><td valign="top"><a href="#set_max_sessions-3">set_max_sessions/3</a></td><td>Set the maximum number of connections allowed to a specific Host:Port.</td></tr>
+<tr><td valign="top"><a href="#show_dest_status-0">show_dest_status/0</a></td><td></td></tr>
 <tr><td valign="top"><a href="#show_dest_status-2">show_dest_status/2</a></td><td>Shows some internal information about load balancing to a
   specified Host:Port.</td></tr>
 <tr><td valign="top"><a href="#spawn_link_worker_process-2">spawn_link_worker_process/2</a></td><td>Same as spawn_worker_process/2 except the the calling process
@@ -320,6 +321,11 @@ send_req/4, send_req/5, send_req/6.
 <p><tt>set_max_sessions(Host::string(), Port::integer(), Max::integer()) -&gt; ok</tt></p>
 </div><p>Set the maximum number of connections allowed to a specific Host:Port.</p>
 
+<h3 class="function"><a name="show_dest_status-0">show_dest_status/0</a></h3>
+<div class="spec">
+<p><tt>show_dest_status() -&gt; any()</tt></p>
+</div>
+
 <h3 class="function"><a name="show_dest_status-2">show_dest_status/2</a></h3>
 <div class="spec">
 <p><tt>show_dest_status(Host, Port) -&gt; any()</tt></p>
@@ -411,6 +417,6 @@ send_req/4, send_req/5, send_req/6.
 <hr>
 
 <div class="navbar"><a name="#navbar_bottom"></a><table width="100%" border="0" cellspacing="0" cellpadding="2" summary="navigation bar"><tr><td><a href="overview-summary.html" target="overviewFrame">Overview</a></td><td><a href="http://www.erlang.org/"><img src="erlang.png" align="right" border="0" alt="erlang logo"></a></td></tr></table></div>
-<p><i>Generated by EDoc, Jul 7 2009, 23:13:24.</i></p>
+<p><i>Generated by EDoc, Jul 29 2009, 18:43:30.</i></p>
 </body>
 </html>

src/ibrowse.erl

@@ -7,7 +7,7 @@
 %%%-------------------------------------------------------------------
 %% @author Chandrashekhar Mullaparthi <chandrashekhar dot mullaparthi at gmail dot com>
 %% @copyright 2005-2009 Chandrashekhar Mullaparthi
-%% @version 1.5.1
+%% @version 1.5.2
 %% @doc The ibrowse application implements an HTTP 1.1 client. This
 %% module implements the API of the HTTP client. There is one named
 %% process called 'ibrowse' which assists in load balancing and maintaining configuration. There is one load balancing process per unique webserver. There is
@@ -98,6 +98,7 @@
 	 trace_on/2,
 	 trace_off/2,
 	 all_trace_off/0,
+	 show_dest_status/0,
 	 show_dest_status/2
 	]).
 
@@ -480,6 +481,44 @@ all_trace_off() ->
     ibrowse ! all_trace_off,
     ok.
 
+show_dest_status() ->
+    Dests = lists:filter(fun({lb_pid, {Host, Port}, _}) when is_list(Host),
+							     is_integer(Port) ->
+				 true;
+			    (_) ->
+				 false
+			 end, ets:tab2list(ibrowse_lb)),
+    All_ets = ets:all(),
+    io:format("~-40.40s | ~-5.5s | ~-10.10s | ~s~n",
+	      ["Server:port", "ETS", "Num conns", "LB Pid"]),
+    io:format("~80.80.=s~n", [""]),
+    lists:foreach(fun({lb_pid, {Host, Port}, Lb_pid}) ->
+			  case lists:dropwhile(
+				 fun(Tid) ->
+					 ets:info(Tid, owner) /= Lb_pid
+				 end, All_ets) of
+			      [] ->
+				  io:format("~40.40s | ~-5.5s | ~-5.5s | ~s~n",
+					    [Host ++ ":" ++ integer_to_list(Port),
+					     "",
+					     "",
+					     io_lib:format("~p", [Lb_pid])]
+					   );
+			      [Tid | _] ->
+				  catch (
+				    begin
+					Size = ets:info(Tid, size),
+					io:format("~40.40s | ~-5.5s | ~-5.5s | ~s~n",
+						  [Host ++ ":" ++ integer_to_list(Port),
+						   integer_to_list(Tid),
+						   integer_to_list(Size),
+						   io_lib:format("~p", [Lb_pid])]
+						 )
+				    end
+				   )
+				  end
+		  end, Dests).
+					  
 %% @doc Shows some internal information about load balancing to a
 %% specified Host:Port. Info about workers spawned using
 %% spawn_worker_process/2 or spawn_link_worker_process/2 is not

src/ibrowse_http_client.erl

@@ -137,7 +137,7 @@ handle_call({send_req, {Url, Headers, Method, Body, Options, Timeout}},
 handle_call(stop, _From, State) ->
     do_close(State),
     do_error_reply(State, closing_on_request),
-    {stop, normal, State};
+    {stop, normal, ok, State};
 
 handle_call(Request, _From, State) ->
     Reply = {unknown_request, Request},
@@ -184,6 +184,15 @@ handle_info({ssl_closed, _Sock}, State) ->
     handle_sock_closed(State),
     {stop, normal, State};
 
+handle_info({tcp_error, _Sock}, State) ->
+    io:format("Error on connection to ~1000.p:~1000.p~n", [State#state.host, State#state.port]),
+    handle_sock_closed(State),
+    {stop, normal, State};
+handle_info({ssl_error, _Sock}, State) ->
+    io:format("Error on SSL connection to ~1000.p:~1000.p~n", [State#state.host, State#state.port]),
+    handle_sock_closed(State),
+    {stop, normal, State};
+
 handle_info({req_timedout, From}, State) ->
     case lists:keysearch(From, #request.from, queue:to_list(State#state.reqs)) of
 	false ->
@@ -204,6 +213,8 @@ handle_info({trace, Bool}, State) ->
     {noreply, State};
 
 handle_info(Info, State) ->
+    io:format("Unknown message recvd for ~1000.p:~1000.p -> ~p~n",
+	      [State#state.host, State#state.port, Info]),
     io:format("Recvd unknown message ~p when in state: ~p~n", [Info, State]),
     {noreply, State}.
 
@@ -869,8 +880,8 @@ is_connection_closing(_, _)                -> false.
 
 %% This clause determines the chunk size when given data from the beginning of the chunk
 parse_11_response(DataRecvd,
-		  #state{transfer_encoding=chunked, 
-			 chunk_size=chunk_start,
+		  #state{transfer_encoding = chunked, 
+			 chunk_size = chunk_start,
 			 chunk_size_buffer = Chunk_sz_buf
 			} = State) ->
     case scan_crlf(Chunk_sz_buf, DataRecvd) of
@@ -906,20 +917,20 @@ parse_11_response(DataRecvd,
 	{yes, _, NextChunk} ->
 	    State_1 = State#state{chunk_size = chunk_start,
 				  chunk_size_buffer = <<>>,
-%%				  reply_buffer = Buf_1,
 				  deleted_crlf = true},
 	    parse_11_response(NextChunk, State_1);
 	{no, Data_1} ->
-%%	    State#state{reply_buffer = Data_1, rep_buf_size = size(Data_1)}
 	    State#state{chunk_size_buffer = Data_1}
     end;
 
-%% This clause deals with the end of a chunked transfer
+%% This clause deals with the end of a chunked transfer. ibrowse does
+%% not support Trailers in the Chunked Transfer encoding. Any trailer
+%% received is silently discarded.
 parse_11_response(DataRecvd,
 		  #state{transfer_encoding = chunked, chunk_size = 0, 
 			 cur_req = CurReq,
 			 deleted_crlf = DelCrlf,
-			 reply_buffer = Trailer, reqs = Reqs}=State) ->
+			 chunk_size_buffer = Trailer, reqs = Reqs}=State) ->
     do_trace("Detected end of chunked transfer...~n", []),
     DataRecvd_1 = case DelCrlf of
 		      false ->
@@ -933,7 +944,7 @@ parse_11_response(DataRecvd,
 	    State_1 = handle_response(CurReq, State#state{reqs = Reqs_1}),
 	    parse_response(Rem, reset_state(State_1));
 	{no, Rem} ->
-	    State#state{reply_buffer = Rem, rep_buf_size = size(Rem), deleted_crlf = false}
+	    State#state{chunk_size_buffer = Rem, deleted_crlf = false}
     end;
 
 %% This clause extracts a chunk, given the size.

src/ibrowse_lb.erl

@@ -108,18 +108,19 @@ spawn_connection(Lb_pid, Url,
 
 %% Update max_sessions in #state with supplied value
 handle_call({spawn_connection, _Url, Max_sess, Max_pipe, _}, _From,
-	    #state{ets_tid = Tid,
-		   num_cur_sessions = Num} = State) 
+	    #state{num_cur_sessions = Num} = State) 
     when Num >= Max_sess ->
-    Reply = find_best_connection(Tid, Max_pipe),
-    {reply, Reply, State#state{max_sessions = Max_sess}};
+    State_1 = maybe_create_ets(State),
+    Reply = find_best_connection(State_1#state.ets_tid, Max_pipe),
+    {reply, Reply, State_1#state{max_sessions = Max_sess}};
 
 handle_call({spawn_connection, Url, _Max_sess, _Max_pipe, SSL_options}, _From,
-	    #state{num_cur_sessions = Cur,
-		   ets_tid = Tid} = State) ->
+	    #state{num_cur_sessions = Cur} = State) ->
+    State_1 = maybe_create_ets(State),
+    Tid = State_1#state.ets_tid,
     {ok, Pid} = ibrowse_http_client:start_link({Tid, Url, SSL_options}),
     ets:insert(Tid, {{1, Pid}, []}),
-    {reply, {ok, Pid}, State#state{num_cur_sessions = Cur + 1}};
+    {reply, {ok, Pid}, State_1#state{num_cur_sessions = Cur + 1}};
 
 handle_call(Request, _From, State) ->
     Reply = {unknown_request, Request},
@@ -145,11 +146,26 @@ handle_cast(_Msg, State) ->
 handle_info({'EXIT', Parent, _Reason}, #state{parent_pid = Parent} = State) ->
     {stop, normal, State};
 
+handle_info({'EXIT', _Pid, _Reason}, #state{ets_tid = undefined} = State) ->
+    {noreply, State};
+
 handle_info({'EXIT', Pid, _Reason},
 	    #state{num_cur_sessions = Cur,
 		   ets_tid = Tid} = State) ->
     ets:match_delete(Tid, {{'_', Pid}, '_'}),
-    {noreply, State#state{num_cur_sessions = Cur - 1}};
+    Cur_1 = Cur - 1,
+    State_1 = case Cur_1 of
+		  0 ->
+		      ets:delete(Tid),
+		      State#state{ets_tid = undefined};
+		  _ ->
+		      State
+	      end,
+    {noreply, State_1#state{num_cur_sessions = Cur_1}};
+
+handle_info({trace, Bool}, #state{ets_tid = undefined} = State) ->
+    put(my_trace_flag, Bool),
+    {noreply, State};
 
 handle_info({trace, Bool}, #state{ets_tid = Tid} = State) ->
     ets:foldl(fun({{_, Pid}, _}, Acc) when is_pid(Pid) ->
@@ -192,3 +208,9 @@ find_best_connection(Tid, Max_pipe) ->
 	_ ->
 	    {error, retry_later}
     end.
+
+maybe_create_ets(#state{ets_tid = undefined} = State) ->
+    Tid = ets:new(ibrowse_lb, [public, ordered_set]),
+    State#state{ets_tid = Tid};
+maybe_create_ets(State) ->
+    State.

src/ibrowse_test.erl

@@ -231,6 +231,7 @@ unit_tests(Options) ->
 	{'DOWN', Ref, _, _, Info} ->
 	    io:format("Test process crashed: ~p~n", [Info])
     after 60000 ->
+	    exit(Pid, kill),
 	    io:format("Timed out waiting for tests to complete~n", [])
     end.
 
@@ -301,6 +302,9 @@ wait_for_resp(Pid) ->
     receive
 	{async_result, Pid, Res} ->
 	    Res;
+	{async_result, Other_pid, _} ->
+	    io:format("~p: Waiting for result from ~p: got from ~p~n", [self(), Pid, Other_pid]),
+	    wait_for_resp(Pid);
 	{'DOWN', _, _, Pid, Reason} ->
 	    {'EXIT', Reason};
 	{'DOWN', _, _, _, _} ->

vsn.mk

@@ -1,2 +1,2 @@
-IBROWSE_VSN = 1.5.1
+IBROWSE_VSN = 1.5.2