From 3aabf5dfdfaf550add837b92303ff233986c7d2e Mon Sep 17 00:00:00 2001 From: Chandrashekhar Mullaparthi Date: Mon, 23 Jan 2012 06:16:32 +0000 Subject: [PATCH] Added get_metrics --- CHANGELOG | 4 ++ CONTRIBUTORS | 1 + README.md | 2 +- src/ibrowse.app.src | 2 +- src/ibrowse.erl | 134 ++++++++++++++++++++++++++------------------ src/ibrowse_lb.erl | 46 +++++++++------ 6 files changed, 115 insertions(+), 74 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index ca74c24..8d9b218 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,5 +1,9 @@ CONTRIBUTIONS & CHANGE HISTORY ============================== +23-01-2012 - v3.0.0 + * Change to the way pipelining works. + * Fixed various issues reported + 13-04-2011 - v2.2.0 * Filipe David Manana added IPv6 support. This is a mjor new feature, Thank you Filipe! diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 6b979cb..8072a48 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -35,4 +35,5 @@ Steve Vinoski Thomas Lindgren Youn?s Hafri fholzhauser (https://github.com/fholzhauser/) +hyperthunk (https://github.com/hyperthunk/) tholschuh (https://github.com/tholschuh/) diff --git a/README.md b/README.md index aaef2c8..57c0986 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ ibrowse is a HTTP client written in erlang. **Comments to:** chandrashekhar.mullaparthi@gmail.com -**Current Version:** 2.2.0 +**Current Version:** 3.0.0 **Latest Version:** git://github.com/cmullaparthi/ibrowse.git diff --git a/src/ibrowse.app.src b/src/ibrowse.app.src index 9d93868..a8857d5 100644 --- a/src/ibrowse.app.src +++ b/src/ibrowse.app.src @@ -1,6 +1,6 @@ {application, ibrowse, [{description, "HTTP client application"}, - {vsn, "2.2.0"}, + {vsn, "3.0.0"}, {modules, [ ibrowse, ibrowse_http_client, ibrowse_app, diff --git a/src/ibrowse.erl b/src/ibrowse.erl index 7c35042..bda0de1 100644 --- a/src/ibrowse.erl +++ b/src/ibrowse.erl @@ -6,7 +6,7 @@ %%% Created : 11 Oct 2003 by Chandrashekhar Mullaparthi %%%------------------------------------------------------------------- %% @author Chandrashekhar Mullaparthi -%% @copyright 2005-2011 Chandrashekhar Mullaparthi +%% @copyright 2005-2012 Chandrashekhar Mullaparthi %% @doc The ibrowse application implements an HTTP 1.1 client in erlang. This %% module implements the API of the HTTP client. There is one named %% process called 'ibrowse' which assists in load balancing and maintaining configuration. There is one load balancing process per unique webserver. There is @@ -98,7 +98,9 @@ all_trace_off/0, show_dest_status/0, show_dest_status/1, - show_dest_status/2 + show_dest_status/2, + get_metrics/0, + get_metrics/2 ]). -ifdef(debug). @@ -137,7 +139,12 @@ start() -> %% @doc Stop the ibrowse process. Useful when testing using the shell. stop() -> - catch gen_server:call(ibrowse, stop). + case catch gen_server:call(ibrowse, stop) of + {'EXIT',{noproc,_}} -> + ok; + Res -> + Res + end. %% @doc This is the basic function to send a HTTP request. %% The Status return value indicates the HTTP status code returned by the webserver @@ -585,42 +592,18 @@ all_trace_off() -> %% about workers spawned using spawn_worker_process/2 or %% spawn_link_worker_process/2 is not included. show_dest_status() -> - Dests = lists:filter(fun({lb_pid, {Host, Port}, _}) when is_list(Host), - is_integer(Port) -> - true; - (_) -> - false - end, ets:tab2list(ibrowse_lb)), - All_ets = ets:all(), io:format("~-40.40s | ~-5.5s | ~-10.10s | ~s~n", ["Server:port", "ETS", "Num conns", "LB Pid"]), io:format("~80.80.=s~n", [""]), - lists:foreach(fun({lb_pid, {Host, Port}, Lb_pid}) -> - case lists:dropwhile( - fun(Tid) -> - ets:info(Tid, owner) /= Lb_pid - end, All_ets) of - [] -> - io:format("~40.40s | ~-5.5s | ~-5.5s | ~s~n", - [Host ++ ":" ++ integer_to_list(Port), - "", - "", - io_lib:format("~p", [Lb_pid])] - ); - [Tid | _] -> - catch ( - begin - Size = ets:info(Tid, size), - io:format("~40.40s | ~-5.5s | ~-5.5s | ~s~n", - [Host ++ ":" ++ integer_to_list(Port), - io_lib:format("~p", [Tid]), - integer_to_list(Size), - io_lib:format("~p", [Lb_pid])] - ) - end - ) - end - end, Dests). + Metrics = get_metrics(), + lists:foreach( + fun({Host, Port, Lb_pid, Tid, Size}) -> + io:format("~40.40s | ~-5.5s | ~-5.5s | ~p~n", + [Host ++ ":" ++ integer_to_list(Port), + integer_to_list(Tid), + integer_to_list(Size), + Lb_pid]) + end, Metrics). show_dest_status(Url) -> #url{host = Host, port = Port} = ibrowse_lib:parse_url(Url), @@ -631,35 +614,76 @@ show_dest_status(Url) -> %% spawn_worker_process/2 or spawn_link_worker_process/2 is not %% included. show_dest_status(Host, Port) -> + case get_metrics(Host, Port) of + {Lb_pid, MsgQueueSize, Tid, Size, + {{First_p_sz, First_speculative_sz}, + {Last_p_sz, Last_speculative_sz}}} -> + io:format("Load Balancer Pid : ~p~n" + "LB process msg q size : ~p~n" + "LB ETS table id : ~p~n" + "Num Connections : ~p~n" + "Smallest pipeline : ~p:~p~n" + "Largest pipeline : ~p:~p~n", + [Lb_pid, MsgQueueSize, Tid, Size, + First_p_sz, First_speculative_sz, + Last_p_sz, Last_speculative_sz]); + _Err -> + io:format("Metrics not available~n", []) + end. + +get_metrics() -> + Dests = lists:filter(fun({lb_pid, {Host, Port}, _}) when is_list(Host), + is_integer(Port) -> + true; + (_) -> + false + end, ets:tab2list(ibrowse_lb)), + All_ets = ets:all(), + lists:map(fun({lb_pid, {Host, Port}, Lb_pid}) -> + case lists:dropwhile( + fun(Tid) -> + ets:info(Tid, owner) /= Lb_pid + end, All_ets) of + [] -> + {Host, Port, Lb_pid, unknown, 0}; + [Tid | _] -> + Size = case catch (ets:info(Tid, size)) of + N when is_integer(N) -> N; + _ -> 0 + end, + {Host, Port, Lb_pid, Tid, Size} + end + end, Dests). + +get_metrics(Host, Port) -> case ets:lookup(ibrowse_lb, {Host, Port}) of [] -> no_active_processes; [#lb_pid{pid = Lb_pid}] -> - io:format("Load Balancer Pid : ~p~n", [Lb_pid]), - io:format("LB process msg q size : ~p~n", [(catch process_info(Lb_pid, message_queue_len))]), + MsgQueueSize = (catch process_info(Lb_pid, message_queue_len)), + %% {Lb_pid, MsgQueueSize, case lists:dropwhile( fun(Tid) -> ets:info(Tid, owner) /= Lb_pid end, ets:all()) of [] -> - io:format("Couldn't locate ETS table for ~p~n", [Lb_pid]); + {Lb_pid, MsgQueueSize, unknown, 0, unknown}; [Tid | _] -> - Tid_rows = [{X, Y, Z} || {X, Y, Z} <- ets:tab2list(Tid), - is_pid(X), is_integer(Y)], - case Tid_rows of - [] -> - io:format("No active connections~n", []); - _ -> - Tid_rows_sorted = lists:keysort(2, Tid_rows), - First = hd(Tid_rows_sorted), - Last = lists:last(Tid_rows_sorted), - Size = length(Tid_rows), - io:format("LB ETS table id : ~p~n", [Tid]), - io:format("Num Connections : ~p~n", [Size]), - {_, First_p_sz, First_speculative_sz} = First, - {_, Last_p_sz, Last_spec_sz} = Last, - io:format("Smallest pipeline : ~p:~p~n", [First_p_sz, First_speculative_sz]), - io:format("Largest pipeline : ~p:~p~n", [Last_p_sz, Last_spec_sz]) + try + Size = ets:info(Tid, size), + case Size of + 0 -> + ok; + _ -> + First = ets:first(Tid), + Last = ets:last(Tid), + [{_, First_p_sz, First_speculative_sz}] = ets:lookup(Tid, First), + [{_, Last_p_sz, Last_speculative_sz}] = ets:lookup(Tid, Last), + {Lb_pid, MsgQueueSize, Tid, Size, + {{First_p_sz, First_speculative_sz}, {Last_p_sz, Last_speculative_sz}}} + end + catch _:_ -> + not_available end end end. diff --git a/src/ibrowse_lb.erl b/src/ibrowse_lb.erl index aac6534..7bf1fb5 100644 --- a/src/ibrowse_lb.erl +++ b/src/ibrowse_lb.erl @@ -36,7 +36,9 @@ port, max_sessions, max_pipeline_size, - num_cur_sessions = 0}). + num_cur_sessions = 0, + proc_state + }). -include("ibrowse.hrl"). @@ -105,6 +107,21 @@ stop(Lb_pid) -> %% {stop, Reason, State} (terminate/2 is called) %%-------------------------------------------------------------------- +handle_call(stop, _From, #state{ets_tid = undefined} = State) -> + gen_server:reply(_From, ok), + {stop, normal, State}; + +handle_call(stop, _From, #state{ets_tid = Tid} = State) -> + ets:foldl(fun({{_, Pid}, _}, Acc) -> + ibrowse_http_client:stop(Pid), + Acc + end, [], Tid), + gen_server:reply(_From, ok), + {stop, normal, State}; + +handle_call(_, _From, #state{proc_state = shutting_down} = State) -> + {reply, {error, shutting_down}, State}; + %% Update max_sessions in #state with supplied value handle_call({spawn_connection, _Url, Max_sess, Max_pipe, _}, _From, #state{num_cur_sessions = Num} = State) @@ -124,18 +141,6 @@ handle_call({spawn_connection, Url, Max_sess, Max_pipe, SSL_options}, _From, max_sessions = Max_sess, max_pipeline_size = Max_pipe}}; -handle_call(stop, _From, #state{ets_tid = undefined} = State) -> - gen_server:reply(_From, ok), - {stop, normal, State}; - -handle_call(stop, _From, #state{ets_tid = Tid} = State) -> - ets:foldl(fun({{_, Pid}, _}, Acc) -> - ibrowse_http_client:stop(Pid), - Acc - end, [], Tid), - gen_server:reply(_From, ok), - {stop, normal, State}; - handle_call(Request, _From, State) -> Reply = {unknown_request, Request}, {reply, Reply, State}. @@ -192,7 +197,16 @@ handle_info({trace, Bool}, #state{ets_tid = Tid} = State) -> {noreply, State}; handle_info(timeout, State) -> - {noreply, State}; + %% We can't shutdown the process immediately because a request + %% might be in flight. So we first remove the entry from the + %% ibrowse_lb ets table, and then shutdown a couple of seconds + %% later + ets:delete(ibrowse_lb, {State#state.host, State#state.port}), + erlang:send_after(2000, self(), shutdown), + {noreply, State#state{proc_state = shutting_down}}; + +handle_info(shutdown, State) -> + {stop, normal, State}; handle_info(_Info, State) -> {noreply, State}. @@ -233,9 +247,7 @@ find_best_connection(Pid, Tid, Max_pipe) -> end. maybe_create_ets(#state{ets_tid = undefined} = State) -> - Tid = ets:new(ibrowse_lb, [public, - {write_concurrency, true}, - {read_concurrency, true}]), + Tid = ets:new(ibrowse_lb, [public, ordered_set]), State#state{ets_tid = Tid}; maybe_create_ets(State) -> State.