diff --git a/README b/README index aa05522..b0e2a7e 100644 --- a/README +++ b/README @@ -1,4 +1,4 @@ -$Id: README,v 1.15 2008/03/27 01:35:49 chandrusf Exp $ +$Id: README,v 1.16 2008/05/21 15:28:11 chandrusf Exp $ ibrowse is a HTTP client. The following are a list of features. - RFC2616 compliant (AFAIK) @@ -22,6 +22,10 @@ Comments to : Chandrashekhar.Mullaparthi@t-mobile.co.uk CONTRIBUTIONS & CHANGE HISTORY ============================== +21-05-2008 - * Fixed bug in reading some options from the ibrowse.conf file. + Reported by Erik Reitsma on the erlyaws mailing list + * Fixed bug when cleaning up closing connections + 27-03-2008 - * Major rewrite of the load balancing feature. Additional module, ibrowse_lb.erl, introduced to achieve this. * Can now get a handle to a connection process which is not part of diff --git a/doc/ibrowse.html b/doc/ibrowse.html index 301dd47..ff286e0 100644 --- a/doc/ibrowse.html +++ b/doc/ibrowse.html @@ -181,10 +181,13 @@ send_req/4, send_req/5, send_req/6.

send_req/4

send_req(Url, Headers, Method::method(), Body::body()) -> response() -

Same as send_req/3. - If a list is specified for the body it has to be a flat list.

+ If a list is specified for the body it has to be a flat list. The body can also be a fun/0 or a fun/1.
+ If fun/0, the connection handling process will repeatdely call the fun until it returns an error or eof. 
Fun() = {ok, Data} | eof

+ If fun/1, the connection handling process will repeatedly call the fun with the supplied state until it returns an error or eof. 
Fun(State) = {ok, Data} | {ok, Data, NewState} | eof

send_req/5

@@ -354,6 +357,6 @@ send_req/4, send_req/5, send_req/6.

Overviewerlang logo
-

Generated by EDoc, Mar 27 2008, 01:03:49.

+

Generated by EDoc, Mar 27 2008, 01:20:55.

diff --git a/priv/ibrowse.conf b/priv/ibrowse.conf index c142601..83412d7 100644 --- a/priv/ibrowse.conf +++ b/priv/ibrowse.conf @@ -15,3 +15,4 @@ %% If SSL is to be used, both the options, is_ssl and ssl_options MUST be specified %% where option() is all options supported in the ssl module +{{options, "www.google.co.uk", 80}, [{proxy_host, "proxy"}, {proxy_port, 8080}, {proxy_user, "cmullaparthi"}, {proxy_password, "20nov99"}]}. diff --git a/src/ibrowse.erl b/src/ibrowse.erl index 2d9d93b..4e6404a 100644 --- a/src/ibrowse.erl +++ b/src/ibrowse.erl @@ -57,7 +57,7 @@ %% driver isn't actually used.

-module(ibrowse). --vsn('$Id: ibrowse.erl,v 1.6 2008/03/27 01:35:50 chandrusf Exp $ '). +-vsn('$Id: ibrowse.erl,v 1.7 2008/05/21 15:28:11 chandrusf Exp $ '). -behaviour(gen_server). %%-------------------------------------------------------------------- @@ -239,10 +239,11 @@ send_req(Url, Headers, Method, Body, Options, Timeout) -> end, Max_sessions = get_max_sessions(Host, Port, Options), Max_pipeline_size = get_max_pipeline_size(Host, Port, Options), + Options_1 = merge_options(Host, Port, Options), {SSLOptions, IsSSL} = - case get_value(is_ssl, Options, false) of + case get_value(is_ssl, Options_1, false) of false -> {[], false}; - true -> {get_value(ssl_options, Options), true} + true -> {get_value(ssl_options, Options_1), true} end, case ibrowse_lb:spawn_connection(Lb_pid, Parsed_url, Max_sessions, @@ -250,7 +251,7 @@ send_req(Url, Headers, Method, Body, Options, Timeout) -> {SSLOptions, IsSSL}) of {ok, Conn_Pid} -> do_send_req(Conn_Pid, Parsed_url, Headers, - Method, Body, Options, Timeout); + Method, Body, Options_1, Timeout); Err -> Err end; @@ -258,6 +259,18 @@ send_req(Url, Headers, Method, Body, Options, Timeout) -> {error, {url_parsing_failed, Err}} end. +merge_options(Host, Port, Options) -> + Config_options = get_config_value({options, Host, Port}, []), + lists:foldl( + fun({Key, Val}, Acc) -> + case lists:keysearch(Key, 1, Options) of + false -> + [{Key, Val} | Acc]; + _ -> + Acc + end + end, Options, Config_options). + get_lb_pid(Url) -> gen_server:call(?MODULE, {get_lb_pid, Url}). @@ -354,8 +367,10 @@ send_req_direct(Conn_pid, Url, Headers, Method, Body, Options) -> %% returned by spawn_worker_process/2 or spawn_link_worker_process/2 send_req_direct(Conn_pid, Url, Headers, Method, Body, Options, Timeout) -> case catch parse_url(Url) of - #url{} = Parsed_url -> - case do_send_req(Conn_pid, Parsed_url, Headers, Method, Body, Options, Timeout) of + #url{host = Host, + port = Port} = Parsed_url -> + Options_1 = merge_options(Host, Port, Options), + case do_send_req(Conn_pid, Parsed_url, Headers, Method, Body, Options_1, Timeout) of {error, {'EXIT', {noproc, _}}} -> {error, worker_is_dead}; Ret -> diff --git a/src/ibrowse_http_client.erl b/src/ibrowse_http_client.erl index f6093e7..9a0e4d3 100644 --- a/src/ibrowse_http_client.erl +++ b/src/ibrowse_http_client.erl @@ -6,7 +6,7 @@ %%% Created : 11 Oct 2003 by Chandrashekhar Mullaparthi %%%------------------------------------------------------------------- -module(ibrowse_http_client). --vsn('$Id: ibrowse_http_client.erl,v 1.17 2008/03/27 01:35:50 chandrusf Exp $ '). +-vsn('$Id: ibrowse_http_client.erl,v 1.18 2008/05/21 15:28:11 chandrusf Exp $ '). -behaviour(gen_server). %%-------------------------------------------------------------------- @@ -1294,7 +1294,7 @@ shutting_down(#state{lb_ets_tid = undefined}) -> ok; shutting_down(#state{lb_ets_tid = Tid, cur_pipeline_size = Sz}) -> - ets:delete(Tid, {Sz, self()}). + catch ets:delete(Tid, {Sz, self()}). inc_pipeline_counter(#state{is_closing = true} = State) -> State; diff --git a/src/ibrowse_test.erl b/src/ibrowse_test.erl index 5806aee..b4429c9 100644 --- a/src/ibrowse_test.erl +++ b/src/ibrowse_test.erl @@ -4,7 +4,7 @@ %%% Created : 14 Oct 2003 by Chandrashekhar Mullaparthi -module(ibrowse_test). --vsn('$Id: ibrowse_test.erl,v 1.2 2008/03/27 01:35:50 chandrusf Exp $ '). +-vsn('$Id: ibrowse_test.erl,v 1.3 2008/05/21 15:28:11 chandrusf Exp $ '). -export([ load_test/3, send_reqs_1/3, @@ -32,6 +32,7 @@ send_reqs_1(Url, NumWorkers, NumReqsPerWorker) -> Start_time = now(), ets:new(pid_table, [named_table, public]), ets:new(ibrowse_test_results, [named_table, public]), + ets:new(ibrowse_errors, [named_table, public, ordered_set]), init_results(), process_flag(trap_exit, true), log_msg("Starting spawning of workers...~n", []), @@ -45,15 +46,20 @@ send_reqs_1(Url, NumWorkers, NumReqsPerWorker) -> log_msg("End time : ~1000.p~n", [calendar:now_to_local_time(End_time)]), Elapsed_time_secs = trunc(timer:now_diff(End_time, Start_time) / 1000000), log_msg("Elapsed : ~p~n", [Elapsed_time_secs]), - log_msg("Reqs/sec : ~p~n", [(NumWorkers*NumReqsPerWorker) / Elapsed_time_secs]). + log_msg("Reqs/sec : ~p~n", [(NumWorkers*NumReqsPerWorker) / Elapsed_time_secs]), + dump_errors(). init_results() -> ets:insert(ibrowse_test_results, {crash, 0}), ets:insert(ibrowse_test_results, {send_failed, 0}), ets:insert(ibrowse_test_results, {other_error, 0}), ets:insert(ibrowse_test_results, {success, 0}), + ets:insert(ibrowse_test_results, {retry_later, 0}), + ets:insert(ibrowse_test_results, {trid_mismatch, 0}), + ets:insert(ibrowse_test_results, {success_no_trid, 0}), ets:insert(ibrowse_test_results, {failed, 0}), - ets:insert(ibrowse_test_results, {timeout, 0}). + ets:insert(ibrowse_test_results, {timeout, 0}), + ets:insert(ibrowse_test_results, {req_id, 0}). spawn_workers(_Url, 0, _) -> ok; @@ -89,19 +95,54 @@ do_send_req(Url, NumReqs) -> do_send_req_1(_Url, 0) -> ets:delete(pid_table, self()); do_send_req_1(Url, NumReqs) -> - case ibrowse:send_req(Url, [], get, [], [], 10000) of - {ok, _Status, _Headers, _Body} -> - ets:update_counter(ibrowse_test_results, success, 1); + Counter = integer_to_list(ets:update_counter(ibrowse_test_results, req_id, 1)), + case ibrowse:send_req(Url, [{"ib_req_id", Counter}], get, [], [], 10000) of + {ok, _Status, Headers, _Body} -> + case lists:keysearch("ib_req_id", 1, Headers) of + {value, {_, Counter}} -> + ets:update_counter(ibrowse_test_results, success, 1); + {value, _} -> + ets:update_counter(ibrowse_test_results, trid_mismatch, 1); + false -> + ets:update_counter(ibrowse_test_results, success_no_trid, 1) + end; {error, req_timedout} -> ets:update_counter(ibrowse_test_results, timeout, 1); {error, send_failed} -> ets:update_counter(ibrowse_test_results, send_failed, 1); - _Err -> + {error, retry_later} -> + ets:update_counter(ibrowse_test_results, retry_later, 1); + Err -> + ets:insert(ibrowse_errors, {now(), Err}), ets:update_counter(ibrowse_test_results, other_error, 1), ok end, do_send_req_1(Url, NumReqs-1). +dump_errors() -> + case ets:info(ibrowse_errors, size) of + 0 -> + ok; + _ -> + {A, B, C} = now(), + Filename = lists:flatten( + io_lib:format("ibrowse_errors_~p_~p_~p.txt" , [A, B, C])), + case file:open(Filename, [write, delayed_write, raw]) of + {ok, Iod} -> + dump_errors(ets:first(ibrowse_errors), Iod); + Err -> + io:format("failed to create file ~s. Reason: ~p~n", [Filename, Err]), + ok + end + end. + +dump_errors('$end_of_table', Iod) -> + file:close(Iod); +dump_errors(Key, Iod) -> + [{_, Term}] = ets:lookup(ibrowse_errors, Key), + file:write(Iod, io_lib:format("~p~n", [Term])), + dump_errors(ets:next(ibrowse_errors, Key), Iod). + %%------------------------------------------------------------------------------ %% Unit Tests %%------------------------------------------------------------------------------ diff --git a/vsn.mk b/vsn.mk index 7848fa3..ce4cd17 100644 --- a/vsn.mk +++ b/vsn.mk @@ -1,2 +1,2 @@ -IBROWSE_VSN = 1.4 +IBROWSE_VSN = 1.4.1