From 6991be297704c231a4db1e0612564f772d972e93 Mon Sep 17 00:00:00 2001 From: Chandrashekhar Mullaparthi Date: Wed, 29 Jul 2009 18:46:47 +0100 Subject: [PATCH] 29-07-2009 - * The ETS table created for load balancing of requests was not being deleted which led to the node not being able to create any more ETS tables if queries were made to many number of webservers. ibrowse now deletes the ETS table it creates once the last connection to a webserver is dropped. Reported by Seth Falcon. * Spurious data being returned at end of body in certain cases of chunked encoded responses from the server. Reported by Chris Newcombe. --- README | 12 ++++++++++- doc/ibrowse.html | 10 +++++++-- src/ibrowse.erl | 41 ++++++++++++++++++++++++++++++++++++- src/ibrowse_http_client.erl | 27 ++++++++++++++++-------- src/ibrowse_lb.erl | 38 ++++++++++++++++++++++++++-------- src/ibrowse_test.erl | 4 ++++ vsn.mk | 2 +- 7 files changed, 113 insertions(+), 21 deletions(-) diff --git a/README b/README index 5f7952d..238b443 100644 --- a/README +++ b/README @@ -18,12 +18,22 @@ ibrowse is available under two different licenses. LGPL and the BSD license. Comments to : Chandrashekhar.Mullaparthi@gmail.com -Version : 1.5.1 +Version : 1.5.2 Latest version : git://github.com/cmullaparthi/ibrowse.git CONTRIBUTIONS & CHANGE HISTORY ============================== +29-07-2009 - * The ETS table created for load balancing of requests was not + being deleted which led to the node not being able to create + any more ETS tables if queries were made to many number of + webservers. ibrowse now deletes the ETS table it creates once the + last connection to a webserver is dropped. + Reported by Seth Falcon. + * Spurious data being returned at end of body in certain cases of + chunked encoded responses from the server. + Reported by Chris Newcombe. + 03-07-2009 - Added option {stream_to, {Pid, once}} which allows the caller to control when it wants to receive more data. If this option is used, the call ibrowse:stream_next(Req_id) should be used diff --git a/doc/ibrowse.html b/doc/ibrowse.html index 8cefb0f..2936446 100644 --- a/doc/ibrowse.html +++ b/doc/ibrowse.html @@ -12,7 +12,7 @@ The ibrowse application implements an HTTP 1.1 client.

Copyright © 2005-2009 Chandrashekhar Mullaparthi

-

Version: 1.5.1

+

Version: 1.5.2

Behaviours: gen_server.

Authors: Chandrashekhar Mullaparthi (chandrashekhar dot mullaparthi at gmail dot com).

@@ -90,6 +90,7 @@ send_req/4, send_req/5, send_req/6.

set_dest/3Deprecated. set_max_pipeline_size/3Set the maximum pipeline size for each connection to a specific Host:Port. set_max_sessions/3Set the maximum number of connections allowed to a specific Host:Port. +show_dest_status/0 show_dest_status/2Shows some internal information about load balancing to a specified Host:Port. spawn_link_worker_process/2Same as spawn_worker_process/2 except the the calling process @@ -320,6 +321,11 @@ send_req/4, send_req/5, send_req/6.

set_max_sessions(Host::string(), Port::integer(), Max::integer()) -> ok

Set the maximum number of connections allowed to a specific Host:Port.

+

show_dest_status/0

+
+

show_dest_status() -> any()

+
+

show_dest_status/2

show_dest_status(Host, Port) -> any()

@@ -411,6 +417,6 @@ send_req/4, send_req/5, send_req/6.

Overviewerlang logo
-

Generated by EDoc, Jul 7 2009, 23:13:24.

+

Generated by EDoc, Jul 29 2009, 18:43:30.

diff --git a/src/ibrowse.erl b/src/ibrowse.erl index 2d0a510..57ae867 100644 --- a/src/ibrowse.erl +++ b/src/ibrowse.erl @@ -7,7 +7,7 @@ %%%------------------------------------------------------------------- %% @author Chandrashekhar Mullaparthi %% @copyright 2005-2009 Chandrashekhar Mullaparthi -%% @version 1.5.1 +%% @version 1.5.2 %% @doc The ibrowse application implements an HTTP 1.1 client. This %% module implements the API of the HTTP client. There is one named %% process called 'ibrowse' which assists in load balancing and maintaining configuration. There is one load balancing process per unique webserver. There is @@ -98,6 +98,7 @@ trace_on/2, trace_off/2, all_trace_off/0, + show_dest_status/0, show_dest_status/2 ]). @@ -480,6 +481,44 @@ all_trace_off() -> ibrowse ! all_trace_off, ok. +show_dest_status() -> + Dests = lists:filter(fun({lb_pid, {Host, Port}, _}) when is_list(Host), + is_integer(Port) -> + true; + (_) -> + false + end, ets:tab2list(ibrowse_lb)), + All_ets = ets:all(), + io:format("~-40.40s | ~-5.5s | ~-10.10s | ~s~n", + ["Server:port", "ETS", "Num conns", "LB Pid"]), + io:format("~80.80.=s~n", [""]), + lists:foreach(fun({lb_pid, {Host, Port}, Lb_pid}) -> + case lists:dropwhile( + fun(Tid) -> + ets:info(Tid, owner) /= Lb_pid + end, All_ets) of + [] -> + io:format("~40.40s | ~-5.5s | ~-5.5s | ~s~n", + [Host ++ ":" ++ integer_to_list(Port), + "", + "", + io_lib:format("~p", [Lb_pid])] + ); + [Tid | _] -> + catch ( + begin + Size = ets:info(Tid, size), + io:format("~40.40s | ~-5.5s | ~-5.5s | ~s~n", + [Host ++ ":" ++ integer_to_list(Port), + integer_to_list(Tid), + integer_to_list(Size), + io_lib:format("~p", [Lb_pid])] + ) + end + ) + end + end, Dests). + %% @doc Shows some internal information about load balancing to a %% specified Host:Port. Info about workers spawned using %% spawn_worker_process/2 or spawn_link_worker_process/2 is not diff --git a/src/ibrowse_http_client.erl b/src/ibrowse_http_client.erl index f22f993..bfeca4a 100644 --- a/src/ibrowse_http_client.erl +++ b/src/ibrowse_http_client.erl @@ -137,7 +137,7 @@ handle_call({send_req, {Url, Headers, Method, Body, Options, Timeout}}, handle_call(stop, _From, State) -> do_close(State), do_error_reply(State, closing_on_request), - {stop, normal, State}; + {stop, normal, ok, State}; handle_call(Request, _From, State) -> Reply = {unknown_request, Request}, @@ -184,6 +184,15 @@ handle_info({ssl_closed, _Sock}, State) -> handle_sock_closed(State), {stop, normal, State}; +handle_info({tcp_error, _Sock}, State) -> + io:format("Error on connection to ~1000.p:~1000.p~n", [State#state.host, State#state.port]), + handle_sock_closed(State), + {stop, normal, State}; +handle_info({ssl_error, _Sock}, State) -> + io:format("Error on SSL connection to ~1000.p:~1000.p~n", [State#state.host, State#state.port]), + handle_sock_closed(State), + {stop, normal, State}; + handle_info({req_timedout, From}, State) -> case lists:keysearch(From, #request.from, queue:to_list(State#state.reqs)) of false -> @@ -204,6 +213,8 @@ handle_info({trace, Bool}, State) -> {noreply, State}; handle_info(Info, State) -> + io:format("Unknown message recvd for ~1000.p:~1000.p -> ~p~n", + [State#state.host, State#state.port, Info]), io:format("Recvd unknown message ~p when in state: ~p~n", [Info, State]), {noreply, State}. @@ -869,8 +880,8 @@ is_connection_closing(_, _) -> false. %% This clause determines the chunk size when given data from the beginning of the chunk parse_11_response(DataRecvd, - #state{transfer_encoding=chunked, - chunk_size=chunk_start, + #state{transfer_encoding = chunked, + chunk_size = chunk_start, chunk_size_buffer = Chunk_sz_buf } = State) -> case scan_crlf(Chunk_sz_buf, DataRecvd) of @@ -906,20 +917,20 @@ parse_11_response(DataRecvd, {yes, _, NextChunk} -> State_1 = State#state{chunk_size = chunk_start, chunk_size_buffer = <<>>, -%% reply_buffer = Buf_1, deleted_crlf = true}, parse_11_response(NextChunk, State_1); {no, Data_1} -> -%% State#state{reply_buffer = Data_1, rep_buf_size = size(Data_1)} State#state{chunk_size_buffer = Data_1} end; -%% This clause deals with the end of a chunked transfer +%% This clause deals with the end of a chunked transfer. ibrowse does +%% not support Trailers in the Chunked Transfer encoding. Any trailer +%% received is silently discarded. parse_11_response(DataRecvd, #state{transfer_encoding = chunked, chunk_size = 0, cur_req = CurReq, deleted_crlf = DelCrlf, - reply_buffer = Trailer, reqs = Reqs}=State) -> + chunk_size_buffer = Trailer, reqs = Reqs}=State) -> do_trace("Detected end of chunked transfer...~n", []), DataRecvd_1 = case DelCrlf of false -> @@ -933,7 +944,7 @@ parse_11_response(DataRecvd, State_1 = handle_response(CurReq, State#state{reqs = Reqs_1}), parse_response(Rem, reset_state(State_1)); {no, Rem} -> - State#state{reply_buffer = Rem, rep_buf_size = size(Rem), deleted_crlf = false} + State#state{chunk_size_buffer = Rem, deleted_crlf = false} end; %% This clause extracts a chunk, given the size. diff --git a/src/ibrowse_lb.erl b/src/ibrowse_lb.erl index 6a3ad3e..5c3e30e 100644 --- a/src/ibrowse_lb.erl +++ b/src/ibrowse_lb.erl @@ -108,18 +108,19 @@ spawn_connection(Lb_pid, Url, %% Update max_sessions in #state with supplied value handle_call({spawn_connection, _Url, Max_sess, Max_pipe, _}, _From, - #state{ets_tid = Tid, - num_cur_sessions = Num} = State) + #state{num_cur_sessions = Num} = State) when Num >= Max_sess -> - Reply = find_best_connection(Tid, Max_pipe), - {reply, Reply, State#state{max_sessions = Max_sess}}; + State_1 = maybe_create_ets(State), + Reply = find_best_connection(State_1#state.ets_tid, Max_pipe), + {reply, Reply, State_1#state{max_sessions = Max_sess}}; handle_call({spawn_connection, Url, _Max_sess, _Max_pipe, SSL_options}, _From, - #state{num_cur_sessions = Cur, - ets_tid = Tid} = State) -> + #state{num_cur_sessions = Cur} = State) -> + State_1 = maybe_create_ets(State), + Tid = State_1#state.ets_tid, {ok, Pid} = ibrowse_http_client:start_link({Tid, Url, SSL_options}), ets:insert(Tid, {{1, Pid}, []}), - {reply, {ok, Pid}, State#state{num_cur_sessions = Cur + 1}}; + {reply, {ok, Pid}, State_1#state{num_cur_sessions = Cur + 1}}; handle_call(Request, _From, State) -> Reply = {unknown_request, Request}, @@ -145,11 +146,26 @@ handle_cast(_Msg, State) -> handle_info({'EXIT', Parent, _Reason}, #state{parent_pid = Parent} = State) -> {stop, normal, State}; +handle_info({'EXIT', _Pid, _Reason}, #state{ets_tid = undefined} = State) -> + {noreply, State}; + handle_info({'EXIT', Pid, _Reason}, #state{num_cur_sessions = Cur, ets_tid = Tid} = State) -> ets:match_delete(Tid, {{'_', Pid}, '_'}), - {noreply, State#state{num_cur_sessions = Cur - 1}}; + Cur_1 = Cur - 1, + State_1 = case Cur_1 of + 0 -> + ets:delete(Tid), + State#state{ets_tid = undefined}; + _ -> + State + end, + {noreply, State_1#state{num_cur_sessions = Cur_1}}; + +handle_info({trace, Bool}, #state{ets_tid = undefined} = State) -> + put(my_trace_flag, Bool), + {noreply, State}; handle_info({trace, Bool}, #state{ets_tid = Tid} = State) -> ets:foldl(fun({{_, Pid}, _}, Acc) when is_pid(Pid) -> @@ -192,3 +208,9 @@ find_best_connection(Tid, Max_pipe) -> _ -> {error, retry_later} end. + +maybe_create_ets(#state{ets_tid = undefined} = State) -> + Tid = ets:new(ibrowse_lb, [public, ordered_set]), + State#state{ets_tid = Tid}; +maybe_create_ets(State) -> + State. diff --git a/src/ibrowse_test.erl b/src/ibrowse_test.erl index ad3e812..5f7f5c9 100644 --- a/src/ibrowse_test.erl +++ b/src/ibrowse_test.erl @@ -231,6 +231,7 @@ unit_tests(Options) -> {'DOWN', Ref, _, _, Info} -> io:format("Test process crashed: ~p~n", [Info]) after 60000 -> + exit(Pid, kill), io:format("Timed out waiting for tests to complete~n", []) end. @@ -301,6 +302,9 @@ wait_for_resp(Pid) -> receive {async_result, Pid, Res} -> Res; + {async_result, Other_pid, _} -> + io:format("~p: Waiting for result from ~p: got from ~p~n", [self(), Pid, Other_pid]), + wait_for_resp(Pid); {'DOWN', _, _, Pid, Reason} -> {'EXIT', Reason}; {'DOWN', _, _, _, _} -> diff --git a/vsn.mk b/vsn.mk index d561ea0..1bc3608 100644 --- a/vsn.mk +++ b/vsn.mk @@ -1,2 +1,2 @@ -IBROWSE_VSN = 1.5.1 +IBROWSE_VSN = 1.5.2