diff --git a/README b/README index 5f7952d..238b443 100644 --- a/README +++ b/README @@ -18,12 +18,22 @@ ibrowse is available under two different licenses. LGPL and the BSD license. Comments to : Chandrashekhar.Mullaparthi@gmail.com -Version : 1.5.1 +Version : 1.5.2 Latest version : git://github.com/cmullaparthi/ibrowse.git CONTRIBUTIONS & CHANGE HISTORY ============================== +29-07-2009 - * The ETS table created for load balancing of requests was not + being deleted which led to the node not being able to create + any more ETS tables if queries were made to many number of + webservers. ibrowse now deletes the ETS table it creates once the + last connection to a webserver is dropped. + Reported by Seth Falcon. + * Spurious data being returned at end of body in certain cases of + chunked encoded responses from the server. + Reported by Chris Newcombe. + 03-07-2009 - Added option {stream_to, {Pid, once}} which allows the caller to control when it wants to receive more data. If this option is used, the call ibrowse:stream_next(Req_id) should be used diff --git a/doc/ibrowse.html b/doc/ibrowse.html index 8cefb0f..2936446 100644 --- a/doc/ibrowse.html +++ b/doc/ibrowse.html @@ -12,7 +12,7 @@ The ibrowse application implements an HTTP 1.1 client.

Copyright © 2005-2009 Chandrashekhar Mullaparthi

-

Version: 1.5.1

+

Version: 1.5.2

Behaviours: gen_server.

Authors: Chandrashekhar Mullaparthi (chandrashekhar dot mullaparthi at gmail dot com).

@@ -90,6 +90,7 @@ send_req/4, send_req/5, send_req/6.

set_dest/3Deprecated. set_max_pipeline_size/3Set the maximum pipeline size for each connection to a specific Host:Port. set_max_sessions/3Set the maximum number of connections allowed to a specific Host:Port. +show_dest_status/0 show_dest_status/2Shows some internal information about load balancing to a specified Host:Port. spawn_link_worker_process/2Same as spawn_worker_process/2 except the the calling process @@ -320,6 +321,11 @@ send_req/4, send_req/5, send_req/6.

set_max_sessions(Host::string(), Port::integer(), Max::integer()) -> ok

Set the maximum number of connections allowed to a specific Host:Port.

+

show_dest_status/0

+
+

show_dest_status() -> any()

+
+

show_dest_status/2

show_dest_status(Host, Port) -> any()

@@ -411,6 +417,6 @@ send_req/4, send_req/5, send_req/6.

Overviewerlang logo
-

Generated by EDoc, Jul 7 2009, 23:13:24.

+

Generated by EDoc, Jul 29 2009, 18:43:30.

diff --git a/src/ibrowse.erl b/src/ibrowse.erl index 2d0a510..57ae867 100644 --- a/src/ibrowse.erl +++ b/src/ibrowse.erl @@ -7,7 +7,7 @@ %%%------------------------------------------------------------------- %% @author Chandrashekhar Mullaparthi %% @copyright 2005-2009 Chandrashekhar Mullaparthi -%% @version 1.5.1 +%% @version 1.5.2 %% @doc The ibrowse application implements an HTTP 1.1 client. This %% module implements the API of the HTTP client. There is one named %% process called 'ibrowse' which assists in load balancing and maintaining configuration. There is one load balancing process per unique webserver. There is @@ -98,6 +98,7 @@ trace_on/2, trace_off/2, all_trace_off/0, + show_dest_status/0, show_dest_status/2 ]). @@ -480,6 +481,44 @@ all_trace_off() -> ibrowse ! all_trace_off, ok. +show_dest_status() -> + Dests = lists:filter(fun({lb_pid, {Host, Port}, _}) when is_list(Host), + is_integer(Port) -> + true; + (_) -> + false + end, ets:tab2list(ibrowse_lb)), + All_ets = ets:all(), + io:format("~-40.40s | ~-5.5s | ~-10.10s | ~s~n", + ["Server:port", "ETS", "Num conns", "LB Pid"]), + io:format("~80.80.=s~n", [""]), + lists:foreach(fun({lb_pid, {Host, Port}, Lb_pid}) -> + case lists:dropwhile( + fun(Tid) -> + ets:info(Tid, owner) /= Lb_pid + end, All_ets) of + [] -> + io:format("~40.40s | ~-5.5s | ~-5.5s | ~s~n", + [Host ++ ":" ++ integer_to_list(Port), + "", + "", + io_lib:format("~p", [Lb_pid])] + ); + [Tid | _] -> + catch ( + begin + Size = ets:info(Tid, size), + io:format("~40.40s | ~-5.5s | ~-5.5s | ~s~n", + [Host ++ ":" ++ integer_to_list(Port), + integer_to_list(Tid), + integer_to_list(Size), + io_lib:format("~p", [Lb_pid])] + ) + end + ) + end + end, Dests). + %% @doc Shows some internal information about load balancing to a %% specified Host:Port. Info about workers spawned using %% spawn_worker_process/2 or spawn_link_worker_process/2 is not diff --git a/src/ibrowse_http_client.erl b/src/ibrowse_http_client.erl index f22f993..bfeca4a 100644 --- a/src/ibrowse_http_client.erl +++ b/src/ibrowse_http_client.erl @@ -137,7 +137,7 @@ handle_call({send_req, {Url, Headers, Method, Body, Options, Timeout}}, handle_call(stop, _From, State) -> do_close(State), do_error_reply(State, closing_on_request), - {stop, normal, State}; + {stop, normal, ok, State}; handle_call(Request, _From, State) -> Reply = {unknown_request, Request}, @@ -184,6 +184,15 @@ handle_info({ssl_closed, _Sock}, State) -> handle_sock_closed(State), {stop, normal, State}; +handle_info({tcp_error, _Sock}, State) -> + io:format("Error on connection to ~1000.p:~1000.p~n", [State#state.host, State#state.port]), + handle_sock_closed(State), + {stop, normal, State}; +handle_info({ssl_error, _Sock}, State) -> + io:format("Error on SSL connection to ~1000.p:~1000.p~n", [State#state.host, State#state.port]), + handle_sock_closed(State), + {stop, normal, State}; + handle_info({req_timedout, From}, State) -> case lists:keysearch(From, #request.from, queue:to_list(State#state.reqs)) of false -> @@ -204,6 +213,8 @@ handle_info({trace, Bool}, State) -> {noreply, State}; handle_info(Info, State) -> + io:format("Unknown message recvd for ~1000.p:~1000.p -> ~p~n", + [State#state.host, State#state.port, Info]), io:format("Recvd unknown message ~p when in state: ~p~n", [Info, State]), {noreply, State}. @@ -869,8 +880,8 @@ is_connection_closing(_, _) -> false. %% This clause determines the chunk size when given data from the beginning of the chunk parse_11_response(DataRecvd, - #state{transfer_encoding=chunked, - chunk_size=chunk_start, + #state{transfer_encoding = chunked, + chunk_size = chunk_start, chunk_size_buffer = Chunk_sz_buf } = State) -> case scan_crlf(Chunk_sz_buf, DataRecvd) of @@ -906,20 +917,20 @@ parse_11_response(DataRecvd, {yes, _, NextChunk} -> State_1 = State#state{chunk_size = chunk_start, chunk_size_buffer = <<>>, -%% reply_buffer = Buf_1, deleted_crlf = true}, parse_11_response(NextChunk, State_1); {no, Data_1} -> -%% State#state{reply_buffer = Data_1, rep_buf_size = size(Data_1)} State#state{chunk_size_buffer = Data_1} end; -%% This clause deals with the end of a chunked transfer +%% This clause deals with the end of a chunked transfer. ibrowse does +%% not support Trailers in the Chunked Transfer encoding. Any trailer +%% received is silently discarded. parse_11_response(DataRecvd, #state{transfer_encoding = chunked, chunk_size = 0, cur_req = CurReq, deleted_crlf = DelCrlf, - reply_buffer = Trailer, reqs = Reqs}=State) -> + chunk_size_buffer = Trailer, reqs = Reqs}=State) -> do_trace("Detected end of chunked transfer...~n", []), DataRecvd_1 = case DelCrlf of false -> @@ -933,7 +944,7 @@ parse_11_response(DataRecvd, State_1 = handle_response(CurReq, State#state{reqs = Reqs_1}), parse_response(Rem, reset_state(State_1)); {no, Rem} -> - State#state{reply_buffer = Rem, rep_buf_size = size(Rem), deleted_crlf = false} + State#state{chunk_size_buffer = Rem, deleted_crlf = false} end; %% This clause extracts a chunk, given the size. diff --git a/src/ibrowse_lb.erl b/src/ibrowse_lb.erl index 6a3ad3e..5c3e30e 100644 --- a/src/ibrowse_lb.erl +++ b/src/ibrowse_lb.erl @@ -108,18 +108,19 @@ spawn_connection(Lb_pid, Url, %% Update max_sessions in #state with supplied value handle_call({spawn_connection, _Url, Max_sess, Max_pipe, _}, _From, - #state{ets_tid = Tid, - num_cur_sessions = Num} = State) + #state{num_cur_sessions = Num} = State) when Num >= Max_sess -> - Reply = find_best_connection(Tid, Max_pipe), - {reply, Reply, State#state{max_sessions = Max_sess}}; + State_1 = maybe_create_ets(State), + Reply = find_best_connection(State_1#state.ets_tid, Max_pipe), + {reply, Reply, State_1#state{max_sessions = Max_sess}}; handle_call({spawn_connection, Url, _Max_sess, _Max_pipe, SSL_options}, _From, - #state{num_cur_sessions = Cur, - ets_tid = Tid} = State) -> + #state{num_cur_sessions = Cur} = State) -> + State_1 = maybe_create_ets(State), + Tid = State_1#state.ets_tid, {ok, Pid} = ibrowse_http_client:start_link({Tid, Url, SSL_options}), ets:insert(Tid, {{1, Pid}, []}), - {reply, {ok, Pid}, State#state{num_cur_sessions = Cur + 1}}; + {reply, {ok, Pid}, State_1#state{num_cur_sessions = Cur + 1}}; handle_call(Request, _From, State) -> Reply = {unknown_request, Request}, @@ -145,11 +146,26 @@ handle_cast(_Msg, State) -> handle_info({'EXIT', Parent, _Reason}, #state{parent_pid = Parent} = State) -> {stop, normal, State}; +handle_info({'EXIT', _Pid, _Reason}, #state{ets_tid = undefined} = State) -> + {noreply, State}; + handle_info({'EXIT', Pid, _Reason}, #state{num_cur_sessions = Cur, ets_tid = Tid} = State) -> ets:match_delete(Tid, {{'_', Pid}, '_'}), - {noreply, State#state{num_cur_sessions = Cur - 1}}; + Cur_1 = Cur - 1, + State_1 = case Cur_1 of + 0 -> + ets:delete(Tid), + State#state{ets_tid = undefined}; + _ -> + State + end, + {noreply, State_1#state{num_cur_sessions = Cur_1}}; + +handle_info({trace, Bool}, #state{ets_tid = undefined} = State) -> + put(my_trace_flag, Bool), + {noreply, State}; handle_info({trace, Bool}, #state{ets_tid = Tid} = State) -> ets:foldl(fun({{_, Pid}, _}, Acc) when is_pid(Pid) -> @@ -192,3 +208,9 @@ find_best_connection(Tid, Max_pipe) -> _ -> {error, retry_later} end. + +maybe_create_ets(#state{ets_tid = undefined} = State) -> + Tid = ets:new(ibrowse_lb, [public, ordered_set]), + State#state{ets_tid = Tid}; +maybe_create_ets(State) -> + State. diff --git a/src/ibrowse_test.erl b/src/ibrowse_test.erl index ad3e812..5f7f5c9 100644 --- a/src/ibrowse_test.erl +++ b/src/ibrowse_test.erl @@ -231,6 +231,7 @@ unit_tests(Options) -> {'DOWN', Ref, _, _, Info} -> io:format("Test process crashed: ~p~n", [Info]) after 60000 -> + exit(Pid, kill), io:format("Timed out waiting for tests to complete~n", []) end. @@ -301,6 +302,9 @@ wait_for_resp(Pid) -> receive {async_result, Pid, Res} -> Res; + {async_result, Other_pid, _} -> + io:format("~p: Waiting for result from ~p: got from ~p~n", [self(), Pid, Other_pid]), + wait_for_resp(Pid); {'DOWN', _, _, Pid, Reason} -> {'EXIT', Reason}; {'DOWN', _, _, _, _} -> diff --git a/vsn.mk b/vsn.mk index d561ea0..1bc3608 100644 --- a/vsn.mk +++ b/vsn.mk @@ -1,2 +1,2 @@ -IBROWSE_VSN = 1.5.1 +IBROWSE_VSN = 1.5.2