Sfoglia il codice sorgente

ibrowse-1.5.0. See README for details

pull/16/head
Chandrashekhar Mullaparthi 16 anni fa
parent
commit
cf294d5c88
8 ha cambiato i file con 688 aggiunte e 316 eliminazioni
  1. +28
    -3
      README
  2. +65
    -30
      doc/ibrowse.html
  3. +3
    -3
      ebin/ibrowse.app
  4. +94
    -16
      src/ibrowse.erl
  5. +378
    -242
      src/ibrowse_http_client.erl
  6. +1
    -8
      src/ibrowse_lb.erl
  7. +118
    -13
      src/ibrowse_test.erl
  8. +1
    -1
      vsn.mk

+ 28
- 3
README Vedi File

@ -1,5 +1,3 @@
$Id: README,v 1.16 2008/05/21 15:28:11 chandrusf Exp $
ibrowse is a HTTP client. The following are a list of features.
- RFC2616 compliant (AFAIK)
- supports GET, POST, OPTIONS, HEAD, PUT, DELETE, TRACE,
@ -18,10 +16,37 @@ ibrowse is a HTTP client. The following are a list of features.
ibrowse is available under two different licenses. LGPL and the BSD license.
Comments to : Chandrashekhar.Mullaparthi@t-mobile.co.uk
Comments to : Chandrashekhar.Mullaparthi@gmail.com
Version : 1.5.0
Latest version : git://github.com/cmullaparthi/ibrowse.git
CONTRIBUTIONS & CHANGE HISTORY
==============================
29-06-2009 - * Fixed following issues reported by Oscar Hellström
- Use {active, once} instead of {active, true}
- Fix 'dodgy' timeout handling
- Use binaries internally instead of lists to reduce memory
consumption on 64 bit platforms. The default response format
is still 'list' to maintain backwards compatibility. Use the
option {response_format, binary} to get responses as binaries.
* Fixed chunking bug (reported by Adam Kocoloski)
* Added new option {inactivity_timeout, Milliseconds} to timeout
requests if no data is received on the link for the specified
interval. Useful when responses are large and links are flaky.
* Added ibrowse:all_trace_off/0 to turn off all tracing
* Change to the way responses to asynchronous requests are
returned. The following messages have been removed.
* {ibrowse_async_response, Req_id, {chunk_start, Chunk_size}}
* {ibrowse_async_response, Req_id, chunk_end}
* Fixed Makefiles as part of Debian packaging
(thanks to Thomas Lindgren)
* Moved repository from Sourceforge to Github
11-06-2009 - * Added option to control size of streamed chunks. Also added
option for the client to receive responses in binary format.
21-05-2008 - * Fixed bug in reading some options from the ibrowse.conf file.
Reported by Erik Reitsma on the erlyaws mailing list
* Fixed bug when cleaning up closing connections

+ 65
- 30
doc/ibrowse.html Vedi File

@ -10,9 +10,9 @@
<h1>Module ibrowse</h1>
<ul class="index"><li><a href="#description">Description</a></li><li><a href="#index">Function Index</a></li><li><a href="#functions">Function Details</a></li></ul>The ibrowse application implements an HTTP 1.1 client.
<p>Copyright © 2005-2008 Chandrashekhar Mullaparthi</p>
<p>Copyright © 2005-2009 Chandrashekhar Mullaparthi</p>
<p><b>Version:</b> 1.4</p>
<p><b>Version:</b> 1.5.0</p>
<p><b>Behaviours:</b> <a href="gen_server.html"><tt>gen_server</tt></a>.</p>
<p><b>Authors:</b> Chandrashekhar Mullaparthi (<a href="mailto:chandrashekhar dot mullaparthi at gmail dot com"><tt>chandrashekhar dot mullaparthi at gmail dot com</tt></a>).</p>
@ -64,7 +64,8 @@ send_req/4, send_req/5, send_req/6.

speed achieved using only erlang has been good enough, so the
driver isn't actually used.</p>
<h2><a name="index">Function Index</a></h2>
<table width="100%" border="1" cellspacing="0" cellpadding="2" summary="function index"><tr><td valign="top"><a href="#code_change-3">code_change/3</a></td><td></td></tr>
<table width="100%" border="1" cellspacing="0" cellpadding="2" summary="function index"><tr><td valign="top"><a href="#all_trace_off-0">all_trace_off/0</a></td><td>Turn Off ALL tracing.</td></tr>
<tr><td valign="top"><a href="#code_change-3">code_change/3</a></td><td></td></tr>
<tr><td valign="top"><a href="#get_config_value-1">get_config_value/1</a></td><td>Internal export.</td></tr>
<tr><td valign="top"><a href="#get_config_value-2">get_config_value/2</a></td><td>Internal export.</td></tr>
<tr><td valign="top"><a href="#handle_call-3">handle_call/3</a></td><td></td></tr>
@ -111,44 +112,49 @@ send_req/4, send_req/5, send_req/6.

<h2><a name="functions">Function Details</a></h2>
<h3 class="function"><a name="all_trace_off-0">all_trace_off/0</a></h3>
<div class="spec">
<p><tt>all_trace_off() -&gt; ok</tt></p>
</div><p>Turn Off ALL tracing</p>
<h3 class="function"><a name="code_change-3">code_change/3</a></h3>
<div class="spec">
<p><tt>code_change() -&gt; term()</tt></p>
<p><tt>code_change(OldVsn, State, Extra) -&gt; any()</tt></p>
</div>
<h3 class="function"><a name="get_config_value-1">get_config_value/1</a></h3>
<div class="spec">
<p><tt>get_config_value() -&gt; term()</tt></p>
<p><tt>get_config_value(Key) -&gt; any()</tt></p>
</div><p>Internal export</p>
<h3 class="function"><a name="get_config_value-2">get_config_value/2</a></h3>
<div class="spec">
<p><tt>get_config_value() -&gt; term()</tt></p>
<p><tt>get_config_value(Key, DefVal) -&gt; any()</tt></p>
</div><p>Internal export</p>
<h3 class="function"><a name="handle_call-3">handle_call/3</a></h3>
<div class="spec">
<p><tt>handle_call() -&gt; term()</tt></p>
<p><tt>handle_call(Request, From, State) -&gt; any()</tt></p>
</div>
<h3 class="function"><a name="handle_cast-2">handle_cast/2</a></h3>
<div class="spec">
<p><tt>handle_cast() -&gt; term()</tt></p>
<p><tt>handle_cast(Msg, State) -&gt; any()</tt></p>
</div>
<h3 class="function"><a name="handle_info-2">handle_info/2</a></h3>
<div class="spec">
<p><tt>handle_info() -&gt; term()</tt></p>
<p><tt>handle_info(Info, State) -&gt; any()</tt></p>
</div>
<h3 class="function"><a name="init-1">init/1</a></h3>
<div class="spec">
<p><tt>init() -&gt; term()</tt></p>
<p><tt>init(X1) -&gt; any()</tt></p>
</div>
<h3 class="function"><a name="rescan_config-0">rescan_config/0</a></h3>
<div class="spec">
<p><tt>rescan_config() -&gt; term()</tt></p>
<p><tt>rescan_config() -&gt; any()</tt></p>
</div><p>Clear current configuration for ibrowse and load from the file
ibrowse.conf in the IBROWSE_EBIN/../priv directory. Current
configuration is cleared only if the ibrowse.conf file is readable
@ -156,7 +162,7 @@ send_req/4, send_req/5, send_req/6.

<h3 class="function"><a name="rescan_config-1">rescan_config/1</a></h3>
<div class="spec">
<p><tt>rescan_config() -&gt; term()</tt></p>
<p><tt>rescan_config(File) -&gt; any()</tt></p>
</div>
<h3 class="function"><a name="send_req-3">send_req/3</a></h3>
@ -193,7 +199,7 @@ send_req/4, send_req/5, send_req/6.

<div class="spec">
<p><tt>send_req(Url::string(), Headers::<a href="#type-headerList">headerList()</a>, Method::<a href="#type-method">method()</a>, Body::<a href="#type-body">body()</a>, Options::<a href="#type-optionList">optionList()</a>) -&gt; <a href="#type-response">response()</a></tt>
<ul class="definitions"><li><tt><a name="type-optionList">optionList()</a> = [<a href="#type-option">option()</a>]</tt></li>
<li><tt><a name="type-option">option()</a> = {max_sessions, integer()} | {max_pipeline_size, integer()} | {trace, <a href="#type-boolean">boolean()</a>} | {is_ssl, <a href="#type-boolean">boolean()</a>} | {ssl_options, [SSLOpt]} | {pool_name, atom()} | {proxy_host, string()} | {proxy_port, integer()} | {proxy_user, string()} | {proxy_password, string()} | {use_absolute_uri, <a href="#type-boolean">boolean()</a>} | {basic_auth, {<a href="#type-username">username()</a>, <a href="#type-password">password()</a>}} | {cookie, string()} | {content_length, integer()} | {content_type, string()} | {save_response_to_file, <a href="#type-srtf">srtf()</a>} | {stream_to, <a href="#type-process">process()</a>} | {http_vsn, {MajorVsn, MinorVsn}} | {host_header, string()} | {transfer_encoding, {chunked, ChunkSize}}</tt></li>
<li><tt><a name="type-option">option()</a> = {max_sessions, integer()} | {response_format, <a href="#type-response_format">response_format()</a>} | {stream_chunk_size, integer()} | {max_pipeline_size, integer()} | {trace, <a href="#type-boolean">boolean()</a>} | {is_ssl, <a href="#type-boolean">boolean()</a>} | {ssl_options, [SSLOpt]} | {pool_name, atom()} | {proxy_host, string()} | {proxy_port, integer()} | {proxy_user, string()} | {proxy_password, string()} | {use_absolute_uri, <a href="#type-boolean">boolean()</a>} | {basic_auth, {<a href="#type-username">username()</a>, <a href="#type-password">password()</a>}} | {cookie, string()} | {content_length, integer()} | {content_type, string()} | {save_response_to_file, <a href="#type-srtf">srtf()</a>} | {stream_to, <a href="#type-process">process()</a>} | {http_vsn, {MajorVsn, MinorVsn}} | {host_header, string()} | {inactivity_timeout, integer()} | {connect_timeout, integer()} | {transfer_encoding, {chunked, ChunkSize}}</tt></li>
<li><tt><a name="type-process">process()</a> = pid() | atom()</tt></li>
<li><tt><a name="type-username">username()</a> = string()</tt></li>
<li><tt><a name="type-password">password()</a> = string()</tt></li>
@ -201,12 +207,13 @@ send_req/4, send_req/5, send_req/6.

<li><tt>ChunkSize = integer()</tt></li>
<li><tt><a name="type-srtf">srtf()</a> = <a href="#type-boolean">boolean()</a> | <a href="#type-filename">filename()</a></tt></li>
<li><tt><a name="type-filename">filename()</a> = string()</tt></li>
<li><tt><a name="type-response_format">response_format()</a> = list | binary</tt></li>
</ul></p>
</div><p>Same as send_req/4.
For a description of SSL Options, look in the ssl manpage. If the
HTTP Version to use is not specified, the default is 1.1.
<br>
<p>The <code>host_header</code> is useful in the case where ibrowse is
<p>The <code>host_header</code> option is useful in the case where ibrowse is
connecting to a component such as <a href="http://www.stunnel.org">stunnel</a> which then sets up a
secure connection to a webserver. In this case, the URL supplied to
ibrowse must have the stunnel host/port details, but that won't
@ -223,7 +230,35 @@ send_req/4, send_req/5, send_req/6.

<li>Whenever an error occurs in the processing of a request, ibrowse will return as much
information as it has, such as HTTP Status Code and HTTP Headers. When this happens, the response
is of the form <code>{error, {Reason, {stat_code, StatusCode}, HTTP_headers}}</code></li>
</ul></p>
<li>The <code>inactivity_timeout</code> option is useful when
dealing with large response bodies and/or slow links. In these
cases, it might be hard to estimate how long a request will take to
complete. In such cases, the client might want to timeout if no
data has been received on the link for a certain time interval.</li>
<li>
The <code>connect_timeout</code> option is to specify how long the
client process should wait for connection establishment. This is
useful in scenarios where connections to servers are usually setup
very fast, but responses might take much longer compared to
connection setup. In such cases, it is better for the calling
process to timeout faster if there is a problem (DNS lookup
delays/failures, network routing issues, etc). The total timeout
value specified for the request will enforced. To illustrate using
an example:
<code>
ibrowse:send_req("http://www.example.com/cgi-bin/request", [], get, [], [{connect_timeout, 100}], 1000).
</code>
In the above invocation, if the connection isn't established within
100 milliseconds, the request will fail with
<code>{error, conn_failed}</code>.<br>
If connection setup succeeds, the total time allowed for the
request to complete will be 1000 milliseconds minus the time taken
for connection setup.
</li>
</ul>
</p>
<h3 class="function"><a name="send_req-6">send_req/6</a></h3>
<div class="spec">
@ -235,31 +270,31 @@ send_req/4, send_req/5, send_req/6.

<h3 class="function"><a name="send_req_direct-4">send_req_direct/4</a></h3>
<div class="spec">
<p><tt>send_req_direct() -&gt; term()</tt></p>
<p><tt>send_req_direct(Conn_pid, Url, Headers, Method) -&gt; any()</tt></p>
</div><p>Same as send_req/3 except that the first argument is the PID
returned by spawn_worker_process/2 or spawn_link_worker_process/2</p>
<h3 class="function"><a name="send_req_direct-5">send_req_direct/5</a></h3>
<div class="spec">
<p><tt>send_req_direct() -&gt; term()</tt></p>
<p><tt>send_req_direct(Conn_pid, Url, Headers, Method, Body) -&gt; any()</tt></p>
</div><p>Same as send_req/4 except that the first argument is the PID
returned by spawn_worker_process/2 or spawn_link_worker_process/2</p>
<h3 class="function"><a name="send_req_direct-6">send_req_direct/6</a></h3>
<div class="spec">
<p><tt>send_req_direct() -&gt; term()</tt></p>
<p><tt>send_req_direct(Conn_pid, Url, Headers, Method, Body, Options) -&gt; any()</tt></p>
</div><p>Same as send_req/5 except that the first argument is the PID
returned by spawn_worker_process/2 or spawn_link_worker_process/2</p>
<h3 class="function"><a name="send_req_direct-7">send_req_direct/7</a></h3>
<div class="spec">
<p><tt>send_req_direct() -&gt; term()</tt></p>
<p><tt>send_req_direct(Conn_pid, Url, Headers, Method, Body, Options, Timeout) -&gt; any()</tt></p>
</div><p>Same as send_req/6 except that the first argument is the PID
returned by spawn_worker_process/2 or spawn_link_worker_process/2</p>
<h3 class="function"><a name="set_dest-3">set_dest/3</a></h3>
<div class="spec">
<p><tt>set_dest() -&gt; term()</tt></p>
<p><tt>set_dest(Host, Port, T) -&gt; any()</tt></p>
</div><p>Deprecated. Use set_max_sessions/3 and set_max_pipeline_size/3
for achieving the same effect.</p>
@ -275,7 +310,7 @@ send_req/4, send_req/5, send_req/6.

<h3 class="function"><a name="show_dest_status-2">show_dest_status/2</a></h3>
<div class="spec">
<p><tt>show_dest_status() -&gt; term()</tt></p>
<p><tt>show_dest_status(Host, Port) -&gt; any()</tt></p>
</div><p>Shows some internal information about load balancing to a
specified Host:Port. Info about workers spawned using
spawn_worker_process/2 or spawn_link_worker_process/2 is not
@ -283,7 +318,7 @@ send_req/4, send_req/5, send_req/6.

<h3 class="function"><a name="spawn_link_worker_process-2">spawn_link_worker_process/2</a></h3>
<div class="spec">
<p><tt>spawn_link_worker_process() -&gt; term()</tt></p>
<p><tt>spawn_link_worker_process(Host, Port) -&gt; any()</tt></p>
</div><p>Same as spawn_worker_process/2 except the the calling process
is linked to the worker process which is spawned.</p>
@ -305,7 +340,7 @@ send_req/4, send_req/5, send_req/6.

<h3 class="function"><a name="start-0">start/0</a></h3>
<div class="spec">
<p><tt>start() -&gt; term()</tt></p>
<p><tt>start() -&gt; any()</tt></p>
</div><p>Starts the ibrowse process without linking. Useful when testing using the shell</p>
<h3 class="function"><a name="start_link-0">start_link/0</a></h3>
@ -315,7 +350,7 @@ send_req/4, send_req/5, send_req/6.

<h3 class="function"><a name="stop-0">stop/0</a></h3>
<div class="spec">
<p><tt>stop() -&gt; term()</tt></p>
<p><tt>stop() -&gt; any()</tt></p>
</div><p>Stop the ibrowse process. Useful when testing using the shell.</p>
<h3 class="function"><a name="stop_worker_process-1">stop_worker_process/1</a></h3>
@ -327,28 +362,28 @@ send_req/4, send_req/5, send_req/6.

<h3 class="function"><a name="terminate-2">terminate/2</a></h3>
<div class="spec">
<p><tt>terminate() -&gt; term()</tt></p>
<p><tt>terminate(Reason, State) -&gt; any()</tt></p>
</div>
<h3 class="function"><a name="trace_off-0">trace_off/0</a></h3>
<div class="spec">
<p><tt>trace_off() -&gt; term()</tt></p>
<p><tt>trace_off() -&gt; any()</tt></p>
</div><p>Turn tracing off for the ibrowse process</p>
<h3 class="function"><a name="trace_off-2">trace_off/2</a></h3>
<div class="spec">
<p><tt>trace_off(Host, Port) -&gt; term()</tt></p>
<p><tt>trace_off(Host, Port) -&gt; ok</tt></p>
</div><p>Turn tracing OFF for all connections to the specified HTTP
server.</p>
<h3 class="function"><a name="trace_on-0">trace_on/0</a></h3>
<div class="spec">
<p><tt>trace_on() -&gt; term()</tt></p>
<p><tt>trace_on() -&gt; any()</tt></p>
</div><p>Turn tracing on for the ibrowse process</p>
<h3 class="function"><a name="trace_on-2">trace_on/2</a></h3>
<div class="spec">
<p><tt>trace_on(Host, Port) -&gt; term()</tt>
<p><tt>trace_on(Host, Port) -&gt; ok</tt>
<ul class="definitions"><li><tt>Host = string()</tt></li>
<li><tt>Port = integer()</tt></li>
</ul></p>
@ -357,6 +392,6 @@ send_req/4, send_req/5, send_req/6.

<hr>
<div class="navbar"><a name="#navbar_bottom"></a><table width="100%" border="0" cellspacing="0" cellpadding="2" summary="navigation bar"><tr><td><a href="overview-summary.html" target="overviewFrame">Overview</a></td><td><a href="http://www.erlang.org/"><img src="erlang.png" align="right" border="0" alt="erlang logo"></a></td></tr></table></div>
<p><i>Generated by EDoc, Mar 27 2008, 01:20:55.</i></p>
<p><i>Generated by EDoc, Jun 30 2009, 23:44:01.</i></p>
</body>
</html>

+ 3
- 3
ebin/ibrowse.app Vedi File

@ -1,12 +1,12 @@
{application, ibrowse,
[{description, "HTTP client application"},
{vsn, "1.4"},
{vsn, "1.5.0"},
{modules, [ ibrowse,
ibrowse_http_client,
ibrowse_app,
ibrowse_sup,
ibrowse_lb,
ibrowse_lib ]},
ibrowse_lib,
ibrowse_lb ]},
{registered, []},
{applications, [kernel,stdlib,sasl]},
{env, []},

+ 94
- 16
src/ibrowse.erl Vedi File

@ -6,8 +6,8 @@
%%% Created : 11 Oct 2003 by Chandrashekhar Mullaparthi <chandrashekhar.mullaparthi@t-mobile.co.uk>
%%%-------------------------------------------------------------------
%% @author Chandrashekhar Mullaparthi <chandrashekhar dot mullaparthi at gmail dot com>
%% @copyright 2005-2008 Chandrashekhar Mullaparthi
%% @version 1.4
%% @copyright 2005-2009 Chandrashekhar Mullaparthi
%% @version 1.5.0
%% @doc The ibrowse application implements an HTTP 1.1 client. This
%% module implements the API of the HTTP client. There is one named
%% process called 'ibrowse' which assists in load balancing and maintaining configuration. There is one load balancing process per unique webserver. There is
@ -57,7 +57,7 @@
%% driver isn't actually used.</p>
-module(ibrowse).
-vsn('$Id: ibrowse.erl,v 1.7 2008/05/21 15:28:11 chandrusf Exp $ ').
-vsn('$Id: ibrowse.erl,v 1.8 2009/07/01 22:43:19 chandrusf Exp $ ').
-behaviour(gen_server).
%%--------------------------------------------------------------------
@ -96,6 +96,7 @@
trace_off/0,
trace_on/2,
trace_off/2,
all_trace_off/0,
show_dest_status/2
]).
@ -105,8 +106,6 @@
-import(ibrowse_lib, [
parse_url/1,
printable_date/0,
get_value/2,
get_value/3,
do_trace/2
]).
@ -114,6 +113,7 @@
-record(state, {trace = false}).
-include("ibrowse.hrl").
-include_lib("stdlib/include/ms_transform.hrl").
-define(DEF_MAX_SESSIONS,10).
-define(DEF_MAX_PIPELINE_SIZE,10).
@ -170,7 +170,7 @@ send_req(Url, Headers, Method, Body) ->
%% For a description of SSL Options, look in the ssl manpage. If the
%% HTTP Version to use is not specified, the default is 1.1.
%% <br/>
%% <p>The <code>host_header</code> is useful in the case where ibrowse is
%% <p>The <code>host_header</code> option is useful in the case where ibrowse is
%% connecting to a component such as <a
%% href="http://www.stunnel.org">stunnel</a> which then sets up a
%% secure connection to a webserver. In this case, the URL supplied to
@ -188,10 +188,40 @@ send_req(Url, Headers, Method, Body) ->
%% <li>Whenever an error occurs in the processing of a request, ibrowse will return as much
%% information as it has, such as HTTP Status Code and HTTP Headers. When this happens, the response
%% is of the form <code>{error, {Reason, {stat_code, StatusCode}, HTTP_headers}}</code></li>
%%
%% <li>The <code>inactivity_timeout</code> option is useful when
%% dealing with large response bodies and/or slow links. In these
%% cases, it might be hard to estimate how long a request will take to
%% complete. In such cases, the client might want to timeout if no
%% data has been received on the link for a certain time interval.</li>
%%
%% <li>
%% The <code>connect_timeout</code> option is to specify how long the
%% client process should wait for connection establishment. This is
%% useful in scenarios where connections to servers are usually setup
%% very fast, but responses might take much longer compared to
%% connection setup. In such cases, it is better for the calling
%% process to timeout faster if there is a problem (DNS lookup
%% delays/failures, network routing issues, etc). The total timeout
%% value specified for the request will enforced. To illustrate using
%% an example:
%% <code>
%% ibrowse:send_req("http://www.example.com/cgi-bin/request", [], get, [], [{connect_timeout, 100}], 1000).
%% </code>
%% In the above invocation, if the connection isn't established within
%% 100 milliseconds, the request will fail with
%% <code>{error, conn_failed}</code>.<br/>
%% If connection setup succeeds, the total time allowed for the
%% request to complete will be 1000 milliseconds minus the time taken
%% for connection setup.
%% </li>
%% </ul>
%%
%% @spec send_req(Url::string(), Headers::headerList(), Method::method(), Body::body(), Options::optionList()) -> response()
%% optionList() = [option()]
%% option() = {max_sessions, integer()} |
%% {response_format,response_format()}|
%% {stream_chunk_size, integer()} |
%% {max_pipeline_size, integer()} |
%% {trace, boolean()} |
%% {is_ssl, boolean()} |
@ -210,8 +240,10 @@ send_req(Url, Headers, Method, Body) ->
%% {stream_to, process()} |
%% {http_vsn, {MajorVsn, MinorVsn}} |
%% {host_header, string()} |
%% {inactivity_timeout, integer()} |
%% {connect_timeout, integer()} |
%% {transfer_encoding, {chunked, ChunkSize}}
%%
%%
%% process() = pid() | atom()
%% username() = string()
%% password() = string()
@ -219,7 +251,7 @@ send_req(Url, Headers, Method, Body) ->
%% ChunkSize = integer()
%% srtf() = boolean() | filename()
%% filename() = string()
%%
%% response_format() = list | binary
send_req(Url, Headers, Method, Body, Options) ->
send_req(Url, Headers, Method, Body, Options, 30000).
@ -230,7 +262,8 @@ send_req(Url, Headers, Method, Body, Options) ->
send_req(Url, Headers, Method, Body, Options, Timeout) ->
case catch parse_url(Url) of
#url{host = Host,
port = Port} = Parsed_url ->
port = Port,
protocol = Protocol} = Parsed_url ->
Lb_pid = case ets:lookup(ibrowse_lb, {Host, Port}) of
[] ->
get_lb_pid(Parsed_url);
@ -241,9 +274,10 @@ send_req(Url, Headers, Method, Body, Options, Timeout) ->
Max_pipeline_size = get_max_pipeline_size(Host, Port, Options),
Options_1 = merge_options(Host, Port, Options),
{SSLOptions, IsSSL} =
case get_value(is_ssl, Options_1, false) of
case (Protocol == https) orelse
get_value(is_ssl, Options_1, false) of
false -> {[], false};
true -> {get_value(ssl_options, Options_1), true}
true -> {get_value(ssl_options, Options_1, []), true}
end,
case ibrowse_lb:spawn_connection(Lb_pid, Parsed_url,
Max_sessions,
@ -310,16 +344,28 @@ set_max_pipeline_size(Host, Port, Max) when is_integer(Max), Max > 0 ->
do_send_req(Conn_Pid, Parsed_url, Headers, Method, Body, Options, Timeout) ->
case catch ibrowse_http_client:send_req(Conn_Pid, Parsed_url,
Headers, Method, Body,
Headers, Method, ensure_bin(Body),
Options, Timeout) of
{'EXIT', {timeout, _}} ->
{error, req_timedout};
{'EXIT', Reason} ->
{error, {'EXIT', Reason}};
{ok, St_code, Headers, Body} = Ret when is_binary(Body) ->
case get_value(response_format, Options, list) of
list ->
{ok, St_code, Headers, binary_to_list(Body)};
binary ->
Ret
end;
Ret ->
Ret
end.
ensure_bin(L) when is_list(L) ->
list_to_binary(L);
ensure_bin(B) when is_binary(B) ->
B.
%% @doc Creates a HTTP client process to the specified Host:Port which
%% is not part of the load balancing pool. This is useful in cases
%% where some requests to a webserver might take a long time whereas
@ -389,17 +435,25 @@ trace_off() ->
%% @doc Turn tracing on for all connections to the specified HTTP
%% server. Host is whatever is specified as the domain name in the URL
%% @spec trace_on(Host, Port) -> term()
%% @spec trace_on(Host, Port) -> ok
%% Host = string()
%% Port = integer()
trace_on(Host, Port) ->
ibrowse ! {trace, true, Host, Port}.
ibrowse ! {trace, true, Host, Port},
ok.
%% @doc Turn tracing OFF for all connections to the specified HTTP
%% server.
%% @spec trace_off(Host, Port) -> term()
%% @spec trace_off(Host, Port) -> ok
trace_off(Host, Port) ->
ibrowse ! {trace, false, Host, Port}.
ibrowse ! {trace, false, Host, Port},
ok.
%% @doc Turn Off ALL tracing
%% @spec all_trace_off() -> ok
all_trace_off() ->
ibrowse ! all_trace_off,
ok.
%% @doc Shows some internal information about load balancing to a
%% specified Host:Port. Info about workers spawned using
@ -577,6 +631,30 @@ handle_cast(_Msg, State) ->
%% {noreply, State, Timeout} |
%% {stop, Reason, State} (terminate/2 is called)
%%--------------------------------------------------------------------
handle_info(all_trace_off, State) ->
Mspec = [{{ibrowse_conf,{trace,'$1','$2'},true},[],[{{'$1','$2'}}]}],
Trace_on_dests = ets:select(ibrowse_conf, Mspec),
Fun = fun(#lb_pid{host_port = {H, P}, pid = Pid}, _) ->
case lists:member({H, P}, Trace_on_dests) of
false ->
ok;
true ->
catch Pid ! {trace, false}
end;
(#client_conn{key = {H, P, Pid}}, _) ->
case lists:member({H, P}, Trace_on_dests) of
false ->
ok;
true ->
catch Pid ! {trace, false}
end;
(_, Acc) ->
Acc
end,
ets:foldl(Fun, undefined, ibrowse_lb),
ets:select_delete(ibrowse_conf, [{{ibrowse_conf,{trace,'$1','$2'},true},[],['true']}]),
{noreply, State};
handle_info({trace, Bool}, State) ->
put(my_trace_flag, Bool),
{noreply, State};

+ 378
- 242
src/ibrowse_http_client.erl
File diff soppresso perché troppo grande
Vedi File


+ 1
- 8
src/ibrowse_lb.erl Vedi File

@ -7,7 +7,7 @@
%%%-------------------------------------------------------------------
-module(ibrowse_lb).
-vsn('$Id: ibrowse_lb.erl,v 1.1 2008/03/27 01:36:21 chandrusf Exp $ ').
-vsn('$Id: ibrowse_lb.erl,v 1.2 2009/07/01 22:43:19 chandrusf Exp $ ').
-author(chandru).
-behaviour(gen_server).
%%--------------------------------------------------------------------
@ -39,13 +39,6 @@
max_pipeline_size,
num_cur_sessions = 0}).
-import(ibrowse_lib, [
parse_url/1,
printable_date/0,
get_value/3
]).
-include("ibrowse.hrl").
%%====================================================================

+ 118
- 13
src/ibrowse_test.erl Vedi File

@ -4,21 +4,23 @@
%%% Created : 14 Oct 2003 by Chandrashekhar Mullaparthi <chandrashekhar.mullaparthi@t-mobile.co.uk>
-module(ibrowse_test).
-vsn('$Id: ibrowse_test.erl,v 1.3 2008/05/21 15:28:11 chandrusf Exp $ ').
-vsn('$Id: ibrowse_test.erl,v 1.4 2009/07/01 22:43:19 chandrusf Exp $ ').
-export([
load_test/3,
send_reqs_1/3,
do_send_req/2,
unit_tests/0,
unit_tests/1,
unit_tests_1/2,
drv_ue_test/0,
drv_ue_test/1,
ue_test/0,
ue_test/1
ue_test/1,
verify_chunked_streaming/0,
verify_chunked_streaming/1,
i_do_async_req_list/4
]).
-import(ibrowse_lib, [printable_date/0]).
%% Use ibrowse:set_max_sessions/3 and ibrowse:set_max_pipeline_size/3 to
%% tweak settings before running the load test. The defaults are 10 and 10.
load_test(Url, NumWorkers, NumReqsPerWorker) when is_list(Url),
@ -46,7 +48,7 @@ send_reqs_1(Url, NumWorkers, NumReqsPerWorker) ->
log_msg("End time : ~1000.p~n", [calendar:now_to_local_time(End_time)]),
Elapsed_time_secs = trunc(timer:now_diff(End_time, Start_time) / 1000000),
log_msg("Elapsed : ~p~n", [Elapsed_time_secs]),
log_msg("Reqs/sec : ~p~n", [(NumWorkers*NumReqsPerWorker) / Elapsed_time_secs]),
log_msg("Reqs/sec : ~p~n", [round(trunc((NumWorkers*NumReqsPerWorker) / Elapsed_time_secs))]),
dump_errors().
init_results() ->
@ -88,7 +90,7 @@ do_wait() ->
do_wait()
end
end.
do_send_req(Url, NumReqs) ->
do_send_req_1(Url, NumReqs).
@ -149,7 +151,7 @@ dump_errors(Key, Iod) ->
-define(TEST_LIST, [{"http://intranet/messenger", get},
{"http://www.google.co.uk", get},
{"http://www.google.com", get},
{"http://www.google.com", options},
{"http://www.google.com", options},
{"http://www.sun.com", get},
{"http://www.oracle.com", get},
{"http://www.bbc.co.uk", get},
@ -172,26 +174,129 @@ dump_errors(Key, Iod) ->
{"http://jigsaw.w3.org/HTTP/400/toolong/", get},
{"http://jigsaw.w3.org/HTTP/300/", get},
{"http://jigsaw.w3.org/HTTP/Basic/", get, [{basic_auth, {"guest", "guest"}}]},
{"http://jigsaw.w3.org/HTTP/CL/", get}
{"http://jigsaw.w3.org/HTTP/CL/", get},
{"http://www.httpwatch.com/httpgallery/chunked/", get}
]).
unit_tests() ->
unit_tests([]).
unit_tests(Options) ->
{Pid, Ref} = erlang:spawn_monitor(?MODULE, unit_tests_1, [self(), Options]),
receive
{done, Pid} ->
ok;
{'DOWN', Ref, _, _, Info} ->
io:format("Test process crashed: ~p~n", [Info])
after 60000 ->
io:format("Timed out waiting for tests to complete~n", [])
end.
unit_tests_1(Parent, Options) ->
lists:foreach(fun({Url, Method}) ->
execute_req(Url, Method, Options);
({Url, Method, X_Opts}) ->
execute_req(Url, Method, X_Opts ++ Options)
end, ?TEST_LIST).
end, ?TEST_LIST),
Parent ! {done, self()}.
verify_chunked_streaming() ->
verify_chunked_streaming([]).
verify_chunked_streaming(Options) ->
Url = "http://www.httpwatch.com/httpgallery/chunked/",
io:format("URL: ~s~n", [Url]),
io:format("Fetching data without streaming...~n", []),
Result_without_streaming = ibrowse:send_req(
Url, [], get, [],
[{response_format, binary} | Options]),
io:format("Fetching data with streaming as list...~n", []),
Async_response_list = do_async_req_list(
Url, get, [{response_format, list} | Options]),
io:format("Fetching data with streaming as binary...~n", []),
Async_response_bin = do_async_req_list(
Url, get, [{response_format, binary} | Options]),
compare_responses(Result_without_streaming, Async_response_list, Async_response_bin).
compare_responses({ok, St_code, _, Body}, {ok, St_code, _, Body}, {ok, St_code, _, Body}) ->
success;
compare_responses({ok, St_code, _, Body_1}, {ok, St_code, _, Body_2}, {ok, St_code, _, Body_3}) ->
case Body_1 of
Body_2 ->
io:format("Body_1 and Body_2 match~n", []);
Body_3 ->
io:format("Body_1 and Body_3 match~n", []);
_ when Body_2 == Body_3 ->
io:format("Body_2 and Body_3 match~n", []);
_ ->
io:format("All three bodies are different!~n", [])
end,
io:format("Body_1 -> ~p~n", [Body_1]),
io:format("Body_2 -> ~p~n", [Body_2]),
io:format("Body_3 -> ~p~n", [Body_3]),
fail_bodies_mismatch;
compare_responses(R1, R2, R3) ->
io:format("R1 -> ~p~n", [R1]),
io:format("R2 -> ~p~n", [R2]),
io:format("R3 -> ~p~n", [R3]),
fail.
%% do_async_req_list(Url) ->
%% do_async_req_list(Url, get).
%% do_async_req_list(Url, Method) ->
%% do_async_req_list(Url, Method, [{stream_to, self()},
%% {stream_chunk_size, 1000}]).
execute_req(Url, Method) ->
execute_req(Url, Method, []).
do_async_req_list(Url, Method, Options) ->
{Pid,_} = erlang:spawn_monitor(?MODULE, i_do_async_req_list,
[self(), Url, Method,
Options ++ [{stream_chunk_size, 1000}]]),
io:format("Spawned process ~p~n", [Pid]),
wait_for_resp(Pid).
wait_for_resp(Pid) ->
receive
{async_result, Pid, Res} ->
Res;
{'DOWN', _, _, Pid, Reason} ->
{'EXIT', Reason};
{'DOWN', _, _, _, _} ->
wait_for_resp(Pid);
Msg ->
io:format("Recvd unknown message: ~p~n", [Msg]),
wait_for_resp(Pid)
after 10000 ->
{error, timeout}
end.
i_do_async_req_list(Parent, Url, Method, Options) ->
Res = ibrowse:send_req(Url, [], Method, [], [{stream_to, self()} | Options]),
case Res of
{ibrowse_req_id, Req_id} ->
Result = wait_for_async_resp(Req_id, undefined, undefined, []),
Parent ! {async_result, self(), Result};
Err ->
Parent ! {async_result, self(), Err}
end.
wait_for_async_resp(Req_id, Acc_Stat_code, Acc_Headers, Body) ->
receive
{ibrowse_async_headers, Req_id, StatCode, Headers} ->
wait_for_async_resp(Req_id, StatCode, Headers, Body);
{ibrowse_async_response_end, Req_id} ->
Body_1 = list_to_binary(lists:reverse(Body)),
{ok, Acc_Stat_code, Acc_Headers, Body_1};
{ibrowse_async_response, Req_id, Data} ->
wait_for_async_resp(Req_id, Acc_Stat_code, Acc_Headers, [Data | Body]);
Err ->
{ok, Acc_Stat_code, Acc_Headers, Err}
end.
execute_req(Url, Method, Options) ->
io:format("~s, ~p: ", [Url, Method]),
io:format("~7.7w, ~50.50s: ", [Method, Url]),
Result = (catch ibrowse:send_req(Url, [], Method, [], Options)),
case Result of
case Result of
{ok, SCode, _H, _B} ->
io:format("Status code: ~p~n", [SCode]);
Err ->

+ 1
- 1
vsn.mk Vedi File

@ -1,2 +1,2 @@
IBROWSE_VSN = 1.4.1
IBROWSE_VSN = 1.5.0

Caricamento…
Annulla
Salva