Browse Source

Release 2.1.0

pull/23/head
Chandrashekhar Mullaparthi 14 years ago
parent
commit
1a08838a3b
11 changed files with 429 additions and 101 deletions
  1. +7
    -0
      Makefile
  2. +75
    -0
      README
  3. +36
    -15
      doc/ibrowse.html
  4. +1
    -1
      ebin/ibrowse.app
  5. +1
    -1
      src/Makefile
  6. +30
    -3
      src/ibrowse.erl
  7. +136
    -65
      src/ibrowse_http_client.erl
  8. +1
    -1
      src/ibrowse_lib.erl
  9. +31
    -14
      src/ibrowse_test.erl
  10. +110
    -0
      test/ibrowse_test_server.erl
  11. +1
    -1
      vsn.mk

+ 7
- 0
Makefile View File

@ -9,3 +9,10 @@ clean:
install: all
mkdir -p $(DESTDIR)/lib/ibrowse-$(IBROWSE_VSN)/
cp -r ebin $(DESTDIR)/lib/ibrowse-$(IBROWSE_VSN)/
test: all
erl -noshell -pa ebin -s ibrowse -s ibrowse_test unit_tests \
-s ibrowse_test verify_chunked_streaming \
-s ibrowse_test test_chunked_streaming_once \
-s erlang halt

+ 75
- 0
README View File

@ -22,8 +22,83 @@ Version : 2.0.1
Latest version : git://github.com/cmullaparthi/ibrowse.git
CONTRIBUTORS
============
The following people have helped maked ibrowse better by reporting bugs,
supplying patches and also asking for new features. Please write to me if you
have contributed and I've missed you out.
In alphabetical order:
Adam Kocoloski
Andrew Tunnell-Jones
Anthony Molinaro
Benoit Chesneau
Chris Newcombe
Dan Kelley
Derek Upham
Eric Merritt
Erik Reitsma
Filipe David Manana
Geoff Cant
Jeroen Koops
João Lopes
Karol Skocik
Kostis Sagonas
Matthew Reilly
Oscar Hellström
Paul J. Davis
Peter Kristensen
Ram Krishnan
Richard Cameron
Sean Hinde
Seth Falcon
Steve Vinoski
Thomas Lindgren
Younès Hafri
tholschuh (https://github.com/tholschuh/)
CONTRIBUTIONS & CHANGE HISTORY
==============================
25-10-2010 - v2.1.0
* Fixed build on OpenSolaris. Bug report and patch from
tholschuh.
http://github.com/cmullaparthi/ibrowse/issues/issue/10
* Fixed behaviour of inactivity_timeout option. Reported by
João Lopes.
http://github.com/cmullaparthi/ibrowse/issues/issue/11
* Prevent atom table pollution when bogus URLs are input to
ibrowse. Bug report by João Lopes.
http://github.com/cmullaparthi/ibrowse/issues/issue/13
* Automatically do Chunked-Transfer encoding of request body
when the body is generated by a fun. Patch provided by
Filipe David Manana.
http://github.com/cmullaparthi/ibrowse/issues/issue/14
* Depending on input options, ibrowse sometimes included multiple
Content-Length headers. Bug reported by Paul J. Davis
http://github.com/cmullaparthi/ibrowse/issues/issue/15
* Deal with webservers which do not provide a Reason-Phrase on the
response Status-Line. Patch provided by Jeroen Koops.
http://github.com/cmullaparthi/ibrowse/issues/issue/16
* Fixed http://github.com/cmullaparthi/ibrowse/issues/issue/17
This was reported by Filipe David Manana.
* Fixed http://github.com/cmullaparthi/ibrowse/issues/issue/19
This was reported by Dan Kelley and Filipe David Manana.
* Added ibrowse:stream_close/1 to close the connection
associated with a certain response stream. Patch provided by
João Lopes.
* Prevent port number being included in the Host header when port
443 is intended. Bug reported by Andrew Tunnell-Jones
24-09-2010 - v2.0.1
* Removed a spurious io:format statement

+ 36
- 15
doc/ibrowse.html View File

@ -9,14 +9,14 @@
<hr>
<h1>Module ibrowse</h1>
<ul class="index"><li><a href="#description">Description</a></li><li><a href="#index">Function Index</a></li><li><a href="#functions">Function Details</a></li></ul>The ibrowse application implements an HTTP 1.1 client.
<ul class="index"><li><a href="#description">Description</a></li><li><a href="#index">Function Index</a></li><li><a href="#functions">Function Details</a></li></ul>The ibrowse application implements an HTTP 1.1 client in erlang.
<p>Copyright © 2005-2010 Chandrashekhar Mullaparthi</p>
<p><b>Version:</b> 2.0.1</p>
<p><b>Version:</b> 2.1.0</p>
<p><b>Behaviours:</b> <a href="gen_server.html"><tt>gen_server</tt></a>.</p>
<p><b>Authors:</b> Chandrashekhar Mullaparthi (<a href="mailto:chandrashekhar dot mullaparthi at gmail dot com"><tt>chandrashekhar dot mullaparthi at gmail dot com</tt></a>).</p>
<h2><a name="description">Description</a></h2><p>The ibrowse application implements an HTTP 1.1 client. This
<h2><a name="description">Description</a></h2><p>The ibrowse application implements an HTTP 1.1 client in erlang. This
module implements the API of the HTTP client. There is one named
process called 'ibrowse' which assists in load balancing and maintaining configuration. There is one load balancing process per unique webserver. There is
one process to handle one TCP connection to a webserver
@ -87,20 +87,24 @@ send_req/4, send_req/5, send_req/6.

<tr><td valign="top"><a href="#set_dest-3">set_dest/3</a></td><td>Deprecated.</td></tr>
<tr><td valign="top"><a href="#set_max_pipeline_size-3">set_max_pipeline_size/3</a></td><td>Set the maximum pipeline size for each connection to a specific Host:Port.</td></tr>
<tr><td valign="top"><a href="#set_max_sessions-3">set_max_sessions/3</a></td><td>Set the maximum number of connections allowed to a specific Host:Port.</td></tr>
<tr><td valign="top"><a href="#show_dest_status-0">show_dest_status/0</a></td><td></td></tr>
<tr><td valign="top"><a href="#show_dest_status-0">show_dest_status/0</a></td><td>Shows some internal information about load balancing.</td></tr>
<tr><td valign="top"><a href="#show_dest_status-2">show_dest_status/2</a></td><td>Shows some internal information about load balancing to a
specified Host:Port.</td></tr>
<tr><td valign="top"><a href="#spawn_link_worker_process-1">spawn_link_worker_process/1</a></td><td>Same as spawn_worker_process/1 except the the calling process
is linked to the worker process which is spawned.</td></tr>
<tr><td valign="top"><a href="#spawn_link_worker_process-2">spawn_link_worker_process/2</a></td><td></td></tr>
<tr><td valign="top"><a href="#spawn_link_worker_process-2">spawn_link_worker_process/2</a></td><td>Same as spawn_worker_process/2 except the the calling process
is linked to the worker process which is spawned.</td></tr>
<tr><td valign="top"><a href="#spawn_worker_process-1">spawn_worker_process/1</a></td><td>Creates a HTTP client process to the specified Host:Port which
is not part of the load balancing pool.</td></tr>
<tr><td valign="top"><a href="#spawn_worker_process-2">spawn_worker_process/2</a></td><td></td></tr>
<tr><td valign="top"><a href="#spawn_worker_process-2">spawn_worker_process/2</a></td><td>Same as spawn_worker_process/1 but takes as input a Host and Port
instead of a URL.</td></tr>
<tr><td valign="top"><a href="#start-0">start/0</a></td><td>Starts the ibrowse process without linking.</td></tr>
<tr><td valign="top"><a href="#start_link-0">start_link/0</a></td><td>Starts the ibrowse process linked to the calling process.</td></tr>
<tr><td valign="top"><a href="#stop-0">stop/0</a></td><td>Stop the ibrowse process.</td></tr>
<tr><td valign="top"><a href="#stop_worker_process-1">stop_worker_process/1</a></td><td>Terminate a worker process spawned using
spawn_worker_process/2 or spawn_link_worker_process/2.</td></tr>
<tr><td valign="top"><a href="#stream_close-1">stream_close/1</a></td><td>Tell ibrowse to close the connection associated with the
specified stream.</td></tr>
<tr><td valign="top"><a href="#stream_next-1">stream_next/1</a></td><td>Tell ibrowse to stream the next chunk of data to the
caller.</td></tr>
<tr><td valign="top"><a href="#terminate-2">terminate/2</a></td><td></td></tr>
@ -245,11 +249,15 @@ send_req/4, send_req/5, send_req/6.

information as it has, such as HTTP Status Code and HTTP Headers. When this happens, the response
is of the form <code>{error, {Reason, {stat_code, StatusCode}, HTTP_headers}}</code></li>
<li>The <code>inactivity_timeout</code> option is useful when
dealing with large response bodies and/or slow links. In these
cases, it might be hard to estimate how long a request will take to
complete. In such cases, the client might want to timeout if no
data has been received on the link for a certain time interval.</li>
<li><p>The <code>inactivity_timeout</code> option is useful when
dealing with large response bodies and/or slow links. In these
cases, it might be hard to estimate how long a request will take to
complete. In such cases, the client might want to timeout if no
data has been received on the link for a certain time interval.</p>
This value is also used to close connections which are not in use for
the specified timeout value.
</li>
<li>
The <code>connect_timeout</code> option is to specify how long the
@ -343,7 +351,9 @@ send_req/4, send_req/5, send_req/6.

<h3 class="function"><a name="show_dest_status-0">show_dest_status/0</a></h3>
<div class="spec">
<p><tt>show_dest_status() -&gt; any()</tt></p>
</div>
</div><p>Shows some internal information about load balancing. Info
about workers spawned using spawn_worker_process/2 or
spawn_link_worker_process/2 is not included.</p>
<h3 class="function"><a name="show_dest_status-2">show_dest_status/2</a></h3>
<div class="spec">
@ -362,7 +372,8 @@ send_req/4, send_req/5, send_req/6.

<h3 class="function"><a name="spawn_link_worker_process-2">spawn_link_worker_process/2</a></h3>
<div class="spec">
<p><tt>spawn_link_worker_process(Host::string(), Port::integer()) -&gt; {ok, pid()}</tt></p>
</div>
</div><p>Same as spawn_worker_process/2 except the the calling process
is linked to the worker process which is spawned.</p>
<h3 class="function"><a name="spawn_worker_process-1">spawn_worker_process/1</a></h3>
<div class="spec">
@ -383,7 +394,8 @@ send_req/4, send_req/5, send_req/6.

<h3 class="function"><a name="spawn_worker_process-2">spawn_worker_process/2</a></h3>
<div class="spec">
<p><tt>spawn_worker_process(Host::string(), Port::integer()) -&gt; {ok, pid()}</tt></p>
</div>
</div><p>Same as spawn_worker_process/1 but takes as input a Host and Port
instead of a URL.</p>
<h3 class="function"><a name="start-0">start/0</a></h3>
<div class="spec">
@ -407,6 +419,15 @@ send_req/4, send_req/5, send_req/6.

spawn_worker_process/2 or spawn_link_worker_process/2. Requests in
progress will get the error response <pre>{error, closing_on_request}</pre></p>
<h3 class="function"><a name="stream_close-1">stream_close/1</a></h3>
<div class="spec">
<p><tt>stream_close(Req_id::<a href="#type-req_id">req_id()</a>) -&gt; ok | {error, unknown_req_id}</tt></p>
</div><p>Tell ibrowse to close the connection associated with the
specified stream. Should be used in conjunction with the
<code>stream_to</code> option. Note that all requests in progress on
the connection which is serving this Req_id will be aborted, and an
error returned.</p>
<h3 class="function"><a name="stream_next-1">stream_next/1</a></h3>
<div class="spec">
<p><tt>stream_next(Req_id::<a href="#type-req_id">req_id()</a>) -&gt; ok | {error, unknown_req_id}</tt></p>
@ -446,6 +467,6 @@ send_req/4, send_req/5, send_req/6.

<hr>
<div class="navbar"><a name="#navbar_bottom"></a><table width="100%" border="0" cellspacing="0" cellpadding="2" summary="navigation bar"><tr><td><a href="overview-summary.html" target="overviewFrame">Overview</a></td><td><a href="http://www.erlang.org/"><img src="erlang.png" align="right" border="0" alt="erlang logo"></a></td></tr></table></div>
<p><i>Generated by EDoc, Sep 24 2010, 06:42:36.</i></p>
<p><i>Generated by EDoc, Nov 10 2010, 06:04:33.</i></p>
</body>
</html>

+ 1
- 1
ebin/ibrowse.app View File

@ -1,6 +1,6 @@
{application, ibrowse,
[{description, "HTTP client application"},
{vsn, "2.0.0"},
{vsn, "2.1.0"},
{modules, [ ibrowse,
ibrowse_http_client,
ibrowse_app,

+ 1
- 1
src/Makefile View File

@ -24,7 +24,7 @@ $(EBIN)/%.beam: %.erl
${ERLC} $(COMPILER_OPTIONS) $(INCLUDE_DIRS) -o ../ebin $<
$(EBIN)/%.app: %.app.src ../vsn.mk Makefile
sed -e s^%IBROWSE_VSN%^$(IBROWSE_VSN)^ \
sed -e s/%IBROWSE_VSN%/$(IBROWSE_VSN)/ \
$< > $@
clean:

+ 30
- 3
src/ibrowse.erl View File

@ -7,8 +7,8 @@
%%%-------------------------------------------------------------------
%% @author Chandrashekhar Mullaparthi <chandrashekhar dot mullaparthi at gmail dot com>
%% @copyright 2005-2010 Chandrashekhar Mullaparthi
%% @version 2.0.1
%% @doc The ibrowse application implements an HTTP 1.1 client. This
%% @version 2.1.0
%% @doc The ibrowse application implements an HTTP 1.1 client in erlang. This
%% module implements the API of the HTTP client. There is one named
%% process called 'ibrowse' which assists in load balancing and maintaining configuration. There is one load balancing process per unique webserver. There is
%% one process to handle one TCP connection to a webserver
@ -87,6 +87,7 @@
send_req_direct/6,
send_req_direct/7,
stream_next/1,
stream_close/1,
set_max_sessions/3,
set_max_pipeline_size/3,
set_dest/3,
@ -201,7 +202,11 @@ send_req(Url, Headers, Method, Body) ->
%% dealing with large response bodies and/or slow links. In these
%% cases, it might be hard to estimate how long a request will take to
%% complete. In such cases, the client might want to timeout if no
%% data has been received on the link for a certain time interval.</li>
%% data has been received on the link for a certain time interval.
%%
%% This value is also used to close connections which are not in use for
%% the specified timeout value.
%% </li>
%%
%% <li>
%% The <code>connect_timeout</code> option is to specify how long the
@ -458,6 +463,8 @@ ensure_bin({Fun, _} = Body) when is_function(Fun) -> Body.
spawn_worker_process(Url) ->
ibrowse_http_client:start(Url).
%% @doc Same as spawn_worker_process/1 but takes as input a Host and Port
%% instead of a URL.
%% @spec spawn_worker_process(Host::string(), Port::integer()) -> {ok, pid()}
spawn_worker_process(Host, Port) ->
ibrowse_http_client:start({Host, Port}).
@ -468,6 +475,8 @@ spawn_worker_process(Host, Port) ->
spawn_link_worker_process(Url) ->
ibrowse_http_client:start_link(Url).
%% @doc Same as spawn_worker_process/2 except the the calling process
%% is linked to the worker process which is spawned.
%% @spec spawn_link_worker_process(Host::string(), Port::integer()) -> {ok, pid()}
spawn_link_worker_process(Host, Port) ->
ibrowse_http_client:start_link({Host, Port}).
@ -524,6 +533,21 @@ stream_next(Req_id) ->
ok
end.
%% @doc Tell ibrowse to close the connection associated with the
%% specified stream. Should be used in conjunction with the
%% <code>stream_to</code> option. Note that all requests in progress on
%% the connection which is serving this Req_id will be aborted, and an
%% error returned.
%% @spec stream_close(Req_id :: req_id()) -> ok | {error, unknown_req_id}
stream_close(Req_id) ->
case ets:lookup(ibrowse_stream, {req_id_pid, Req_id}) of
[] ->
{error, unknown_req_id};
[{_, Pid}] ->
catch Pid ! {stream_close, Req_id},
ok
end.
%% @doc Turn tracing on for the ibrowse process
trace_on() ->
ibrowse ! {trace, true}.
@ -553,6 +577,9 @@ all_trace_off() ->
ibrowse ! all_trace_off,
ok.
%% @doc Shows some internal information about load balancing. Info
%% about workers spawned using spawn_worker_process/2 or
%% spawn_link_worker_process/2 is not included.
show_dest_status() ->
Dests = lists:filter(fun({lb_pid, {Host, Port}, _}) when is_list(Host),
is_integer(Port) ->

+ 136
- 65
src/ibrowse_http_client.erl View File

@ -37,6 +37,7 @@
-include("ibrowse.hrl").
-record(state, {host, port, connect_timeout,
inactivity_timer_ref,
use_proxy = false, proxy_auth_digest,
ssl_options = [], is_ssl = false, socket,
proxy_tunnel_setup = false,
@ -192,6 +193,12 @@ handle_info({stream_next, Req_id}, #state{socket = Socket,
handle_info({stream_next, _Req_id}, State) ->
{noreply, State};
handle_info({stream_close, _Req_id}, State) ->
shutting_down(State),
do_close(State),
do_error_reply(State, closing_on_request),
{stop, normal, ok, State};
handle_info({tcp_closed, _Sock}, State) ->
do_trace("TCP connection closed by peer!~n", []),
handle_sock_closed(State),
@ -221,6 +228,7 @@ handle_info({req_timedout, From}, State) ->
end;
handle_info(timeout, State) ->
do_trace("Inactivity timeout triggered. Shutting down connection~n", []),
shutting_down(State),
do_error_reply(State, req_timedout),
{stop, normal, State};
@ -273,8 +281,8 @@ handle_sock_data(Data, #state{status = get_header}=State) ->
{stop, normal, State};
State_1 ->
active_once(State_1),
set_inac_timer(State_1),
{noreply, State_1}
State_2 = set_inac_timer(State_1),
{noreply, State_2}
end;
handle_sock_data(Data, #state{status = get_body,
@ -293,8 +301,8 @@ handle_sock_data(Data, #state{status = get_body,
{stop, normal, State};
State_1 ->
active_once(State_1),
set_inac_timer(State_1),
{noreply, State_1}
State_2 = set_inac_timer(State_1),
{noreply, State_2}
end;
_ ->
case parse_11_response(Data, State) of
@ -314,12 +322,12 @@ handle_sock_data(Data, #state{status = get_body,
active_once(State_1)
end,
State_2 = State_1#state{interim_reply_sent = false},
set_inac_timer(State_2),
{noreply, State_2};
State_3 = set_inac_timer(State_2),
{noreply, State_3};
State_1 ->
active_once(State_1),
set_inac_timer(State_1),
{noreply, State_1}
State_2 = set_inac_timer(State_1),
{noreply, State_2}
end
end.
@ -507,29 +515,37 @@ do_send(Req, #state{socket = Sock, is_ssl = false}) -> gen_tcp:send(Sock, Req).
%% {fun_arity_0} |
%% {fun_arity_1, term()}
%% error() = term()
do_send_body(Source, State) when is_function(Source) ->
do_send_body({Source}, State);
do_send_body({Source}, State) when is_function(Source) ->
do_send_body1(Source, Source(), State);
do_send_body({Source, Source_state}, State) when is_function(Source) ->
do_send_body1(Source, Source(Source_state), State);
do_send_body(Body, State) ->
do_send_body(Source, State, TE) when is_function(Source) ->
do_send_body({Source}, State, TE);
do_send_body({Source}, State, TE) when is_function(Source) ->
do_send_body1(Source, Source(), State, TE);
do_send_body({Source, Source_state}, State, TE) when is_function(Source) ->
do_send_body1(Source, Source(Source_state), State, TE);
do_send_body(Body, State, _TE) ->
do_send(Body, State).
do_send_body1(Source, Resp, State) ->
do_send_body1(Source, Resp, State, TE) ->
case Resp of
{ok, Data} ->
do_send(Data, State),
do_send_body({Source}, State);
do_send(maybe_chunked_encode(Data, TE), State),
do_send_body({Source}, State, TE);
{ok, Data, New_source_state} ->
do_send(Data, State),
do_send_body({Source, New_source_state}, State);
do_send(maybe_chunked_encode(Data, TE), State),
do_send_body({Source, New_source_state}, State, TE);
eof when TE == true ->
do_send(<<"0\r\n\r\n">>, State),
ok;
eof ->
ok;
Err ->
Err
end.
maybe_chunked_encode(Data, false) ->
Data;
maybe_chunked_encode(Data, true) ->
[ibrowse_lib:dec2hex(4, size(to_binary(Data))), "\r\n", Data, "\r\n"].
do_close(#state{socket = undefined}) -> ok;
do_close(#state{socket = Sock,
is_ssl = true,
@ -619,11 +635,13 @@ send_req_1(From,
{Req, Body_1} = make_request(connect, Pxy_auth_headers,
Path, Path,
[], Options, State_1),
TE = is_chunked_encoding_specified(Options),
trace_request(Req),
case do_send(Req, State) of
ok ->
case do_send_body(Body_1, State_1) of
case do_send_body(Body_1, State_1, TE) of
ok ->
trace_request_body(Body_1),
active_once(State_1),
Ref = case Timeout of
infinity ->
@ -636,8 +654,8 @@ send_req_1(From,
send_timer = Ref,
proxy_tunnel_setup = in_progress,
tunnel_setup_queue = [{From, Url, Headers, Method, Body, Options, Timeout}]},
set_inac_timer(State_1),
{noreply, State_2};
State_3 = set_inac_timer(State_2),
{noreply, State_3};
Err ->
shutting_down(State_1),
do_trace("Send failed... Reason: ~p~n", [Err]),
@ -706,10 +724,12 @@ send_req_1(From,
AbsPath, RelPath, Body, Options, State_1),
trace_request(Req),
do_setopts(Socket, Caller_socket_options, Is_ssl),
TE = is_chunked_encoding_specified(Options),
case do_send(Req, State_1) of
ok ->
case do_send_body(Body_1, State_1) of
case do_send_body(Body_1, State_1, TE) of
ok ->
trace_request_body(Body_1),
State_2 = inc_pipeline_counter(State_1),
active_once(State_2),
Ref = case Timeout of
@ -732,8 +752,8 @@ send_req_1(From,
_ ->
gen_server:reply(From, {ibrowse_req_id, ReqId})
end,
set_inac_timer(State_1),
{noreply, State_3};
State_4 = set_inac_timer(State_3),
{noreply, State_4};
Err ->
shutting_down(State_1),
do_trace("Send failed... Reason: ~p~n", [Err]),
@ -759,6 +779,7 @@ maybe_modify_headers(#url{host = Host, port = Port} = Url,
false ->
case Port of
80 -> Host;
443 -> Host;
_ -> [Host, ":", integer_to_list(Port)]
end;
{value, {_, Host_h_val}} ->
@ -802,31 +823,42 @@ http_auth_digest(Username, Password) ->
make_request(Method, Headers, AbsPath, RelPath, Body, Options,
#state{use_proxy = UseProxy, is_ssl = Is_ssl}) ->
HttpVsn = http_vsn_string(get_value(http_vsn, Options, {1,1})),
Fun1 = fun({X, Y}) when is_atom(X) ->
{to_lower(atom_to_list(X)), X, Y};
({X, Y}) when is_list(X) ->
{to_lower(X), X, Y}
end,
Headers_0 = [Fun1(X) || X <- Headers],
Headers_1 =
case get_value(content_length, Headers, false) of
false when (Body == []) or
(Body == <<>>) or
is_tuple(Body) or
is_function(Body) ->
Headers;
case lists:keysearch("content-length", 1, Headers_0) of
false when (Body == []) orelse
(Body == <<>>) orelse
is_tuple(Body) orelse
is_function(Body) ->
Headers_0;
false when is_binary(Body) ->
[{"content-length", integer_to_list(size(Body))} | Headers];
false ->
[{"content-length", integer_to_list(length(Body))} | Headers];
[{"content-length", "content-length", integer_to_list(size(Body))} | Headers_0];
false when is_list(Body) ->
[{"content-length", "content-length", integer_to_list(length(Body))} | Headers_0];
_ ->
Headers
%% Content-Length is already specified
Headers_0
end,
{Headers_2, Body_1} =
case get_value(transfer_encoding, Options, false) of
case is_chunked_encoding_specified(Options) of
false ->
{Headers_1, Body};
{chunked, ChunkSize} ->
{[{X, Y} || {X, Y} <- Headers_1,
X /= "Content-Length",
X /= "content-length",
X /= content_length] ++
{[{Y, Z} || {_, Y, Z} <- Headers_1], Body};
true ->
Chunk_size_1 = case get_value(transfer_encoding, Options) of
chunked ->
5120;
{chunked, Chunk_size} ->
Chunk_size
end,
{[{Y, Z} || {X, Y, Z} <- Headers_1,
X /= "content-length"] ++
[{"Transfer-Encoding", "chunked"}],
chunk_request_body(Body, ChunkSize)}
chunk_request_body(Body, Chunk_size_1)}
end,
Headers_3 = cons_headers(Headers_2),
Uri = case get_value(use_absolute_uri, Options, false) or UseProxy of
@ -842,6 +874,16 @@ make_request(Method, Headers, AbsPath, RelPath, Body, Options,
end,
{[method(Method), " ", Uri, " ", HttpVsn, crnl(), Headers_3, crnl()], Body_1}.
is_chunked_encoding_specified(Options) ->
case get_value(transfer_encoding, Options, false) of
false ->
false;
{chunked, _} ->
true;
chunked ->
true
end.
http_vsn_string({0,9}) -> "HTTP/0.9";
http_vsn_string({1,0}) -> "HTTP/1.0";
http_vsn_string({1,1}) -> "HTTP/1.1".
@ -873,6 +915,9 @@ encode_headers([{Name,Val} | T], Acc) when is_atom(Name) ->
encode_headers([], Acc) ->
lists:reverse(Acc).
chunk_request_body(Body, _ChunkSize) when is_tuple(Body) orelse
is_function(Body) ->
Body;
chunk_request_body(Body, ChunkSize) ->
chunk_request_body(Body, ChunkSize, []).
@ -1060,7 +1105,7 @@ upgrade_to_ssl(#state{socket = Socket,
send_queued_requests([], State) ->
do_trace("Sent all queued requests via SSL connection~n", []),
State#state{tunnel_setup_queue = done};
State#state{tunnel_setup_queue = []};
send_queued_requests([{From, Url, Headers, Method, Body, Options, Timeout} | Q],
State) ->
case send_req_1(From, Url, Headers, Method, Body, Options, Timeout, State) of
@ -1217,7 +1262,6 @@ handle_response(#request{from=From, stream_to=StreamTo, req_id=ReqId,
reply_buffer = RepBuf,
recvd_headers = RespHeaders}=State) when SaveResponseToFile /= false ->
Body = RepBuf,
State_1 = set_cur_request(State),
file:close(Fd),
ResponseBody = case TmpFilename of
undefined ->
@ -1232,9 +1276,9 @@ handle_response(#request{from=From, stream_to=StreamTo, req_id=ReqId,
false ->
{ok, SCode, Resp_headers_1, ResponseBody}
end,
State_2 = do_reply(State_1, From, StreamTo, ReqId, Resp_format, Reply),
State_1 = do_reply(State, From, StreamTo, ReqId, Resp_format, Reply),
cancel_timer(ReqTimer, {eat_message, {req_timedout, From}}),
State_2;
set_cur_request(State_1);
handle_response(#request{from=From, stream_to=StreamTo, req_id=ReqId,
response_format = Resp_format,
options = Options},
@ -1245,7 +1289,6 @@ handle_response(#request{from=From, stream_to=StreamTo, req_id=ReqId,
reply_buffer = RepBuf,
send_timer = ReqTimer} = State) ->
Body = RepBuf,
%% State_1 = set_cur_request(State),
{Resp_headers_1, Raw_headers_1} = maybe_add_custom_headers(Resp_headers, Raw_headers, Options),
Reply = case get_value(give_raw_headers, Options, false) of
true ->
@ -1253,15 +1296,8 @@ handle_response(#request{from=From, stream_to=StreamTo, req_id=ReqId,
false ->
{ok, SCode, Resp_headers_1, Body}
end,
State_1 = case get(conn_close) of
"close" ->
do_reply(State, From, StreamTo, ReqId, Resp_format, Reply),
exit(normal);
_ ->
State_1_1 = do_reply(State, From, StreamTo, ReqId, Resp_format, Reply),
cancel_timer(ReqTimer, {eat_message, {req_timedout, From}}),
State_1_1
end,
State_1 = do_reply(State, From, StreamTo, ReqId, Resp_format, Reply),
cancel_timer(ReqTimer, {eat_message, {req_timedout, From}}),
set_cur_request(State_1).
reset_state(State) ->
@ -1353,6 +1389,8 @@ parse_status_line([32 | T], get_prot_vsn, ProtVsn, StatCode) ->
parse_status_line(T, get_status_code, ProtVsn, StatCode);
parse_status_line([32 | T], get_status_code, ProtVsn, StatCode) ->
{ok, lists:reverse(ProtVsn), lists:reverse(StatCode), T};
parse_status_line([], get_status_code, ProtVsn, StatCode) ->
{ok, lists:reverse(ProtVsn), lists:reverse(StatCode), []};
parse_status_line([H | T], get_prot_vsn, ProtVsn, StatCode) ->
parse_status_line(T, get_prot_vsn, [H|ProtVsn], StatCode);
parse_status_line([H | T], get_status_code, ProtVsn, StatCode) ->
@ -1710,28 +1748,61 @@ get_stream_chunk_size(Options) ->
end.
set_inac_timer(State) ->
set_inac_timer(State, get_inac_timeout(State)).
cancel_timer(State#state.inactivity_timer_ref),
set_inac_timer(State#state{inactivity_timer_ref = undefined},
get_inac_timeout(State)).
set_inac_timer(_State, Timeout) when is_integer(Timeout) ->
erlang:send_after(Timeout, self(), timeout);
set_inac_timer(_, _) ->
undefined.
set_inac_timer(State, Timeout) when is_integer(Timeout) ->
Ref = erlang:send_after(Timeout, self(), timeout),
State#state{inactivity_timer_ref = Ref};
set_inac_timer(State, _) ->
State.
get_inac_timeout(#state{cur_req = #request{options = Opts}}) ->
get_value(inactivity_timeout, Opts, infinity);
get_inac_timeout(#state{cur_req = undefined}) ->
infinity.
case ibrowse:get_config_value(inactivity_timeout, undefined) of
Val when is_integer(Val) ->
Val;
_ ->
case application:get_env(ibrowse, inactivity_timeout) of
{ok, Val} when is_integer(Val), Val > 0 ->
Val;
_ ->
10000
end
end.
trace_request(Req) ->
case get(my_trace_flag) of
true ->
%%Avoid the binary operations if trace is not on...
NReq = binary_to_list(list_to_binary(Req)),
NReq = to_binary(Req),
do_trace("Sending request: ~n"
"--- Request Begin ---~n~s~n"
"--- Request End ---~n", [NReq]);
_ -> ok
end.
trace_request_body(Body) ->
case get(my_trace_flag) of
true ->
%%Avoid the binary operations if trace is not on...
NBody = to_binary(Body),
case size(NBody) > 1024 of
true ->
ok;
false ->
do_trace("Sending request body: ~n"
"--- Request Body Begin ---~n~s~n"
"--- Request Body End ---~n", [NBody])
end;
false ->
ok
end.
to_integer(X) when is_list(X) -> list_to_integer(X);
to_integer(X) when is_integer(X) -> X.
to_binary(X) when is_list(X) -> list_to_binary(X);
to_binary(X) when is_binary(X) -> X.

+ 1
- 1
src/ibrowse_lib.erl View File

@ -208,7 +208,7 @@ parse_url(Url) ->
parse_url([$:, $/, $/ | _], get_protocol, Url, []) ->
{invalid_uri_1, Url};
parse_url([$:, $/, $/ | T], get_protocol, Url, TmpAcc) ->
Prot = list_to_atom(lists:reverse(TmpAcc)),
Prot = list_to_existing_atom(lists:reverse(TmpAcc)),
parse_url(T, get_username,
Url#url{protocol = Prot},
[]);

+ 31
- 14
src/ibrowse_test.erl View File

@ -217,14 +217,18 @@ dump_errors(Key, Iod) ->
{"http://jigsaw.w3.org/HTTP/300/", get},
{"http://jigsaw.w3.org/HTTP/Basic/", get, [{basic_auth, {"guest", "guest"}}]},
{"http://jigsaw.w3.org/HTTP/CL/", get},
{"http://www.httpwatch.com/httpgallery/chunked/", get}
{"http://www.httpwatch.com/httpgallery/chunked/", get},
{"https://github.com", get, [{ssl_options, [{depth, 2}]}]}
]).
unit_tests() ->
unit_tests([]).
unit_tests(Options) ->
application:start(crypto),
application:start(public_key),
application:start(ssl),
ibrowse:start(),
Options_1 = Options ++ [{connect_timeout, 5000}],
{Pid, Ref} = erlang:spawn_monitor(?MODULE, unit_tests_1, [self(), Options_1]),
receive
@ -249,32 +253,45 @@ verify_chunked_streaming() ->
verify_chunked_streaming([]).
verify_chunked_streaming(Options) ->
io:format("~nVerifying that chunked streaming is working...~n", []),
Url = "http://www.httpwatch.com/httpgallery/chunked/",
io:format("URL: ~s~n", [Url]),
io:format("Fetching data without streaming...~n", []),
io:format(" URL: ~s~n", [Url]),
io:format(" Fetching data without streaming...~n", []),
Result_without_streaming = ibrowse:send_req(
Url, [], get, [],
[{response_format, binary} | Options]),
io:format("Fetching data with streaming as list...~n", []),
io:format(" Fetching data with streaming as list...~n", []),
Async_response_list = do_async_req_list(
Url, get, [{response_format, list} | Options]),
io:format("Fetching data with streaming as binary...~n", []),
io:format(" Fetching data with streaming as binary...~n", []),
Async_response_bin = do_async_req_list(
Url, get, [{response_format, binary} | Options]),
io:format("Fetching data with streaming as binary, {active, once}...~n", []),
io:format(" Fetching data with streaming as binary, {active, once}...~n", []),
Async_response_bin_once = do_async_req_list(
Url, get, [once, {response_format, binary} | Options]),
compare_responses(Result_without_streaming, Async_response_list, Async_response_bin),
compare_responses(Result_without_streaming, Async_response_list, Async_response_bin_once).
Res1 = compare_responses(Result_without_streaming, Async_response_list, Async_response_bin),
Res2 = compare_responses(Result_without_streaming, Async_response_list, Async_response_bin_once),
case {Res1, Res2} of
{success, success} ->
io:format(" Chunked streaming working~n", []);
_ ->
ok
end.
test_chunked_streaming_once() ->
test_chunked_streaming_once([]).
test_chunked_streaming_once(Options) ->
io:format("~nTesting chunked streaming with the {stream_to, {Pid, once}} option...~n", []),
Url = "http://www.httpwatch.com/httpgallery/chunked/",
io:format("URL: ~s~n", [Url]),
io:format("Fetching data with streaming as binary, {active, once}...~n", []),
do_async_req_list(Url, get, [once, {response_format, binary} | Options]).
io:format(" URL: ~s~n", [Url]),
io:format(" Fetching data with streaming as binary, {active, once}...~n", []),
case do_async_req_list(Url, get, [once, {response_format, binary} | Options]) of
{ok, _, _, _} ->
io:format(" Success!~n", []);
Err ->
io:format(" Fail: ~p~n", [Err])
end.
compare_responses({ok, St_code, _, Body}, {ok, St_code, _, Body}, {ok, St_code, _, Body}) ->
success;
@ -310,7 +327,7 @@ do_async_req_list(Url, Method, Options) ->
{Pid,_} = erlang:spawn_monitor(?MODULE, i_do_async_req_list,
[self(), Url, Method,
Options ++ [{stream_chunk_size, 1000}]]),
io:format("Spawned process ~p~n", [Pid]),
%% io:format("Spawned process ~p~n", [Pid]),
wait_for_resp(Pid).
wait_for_resp(Pid) ->
@ -354,7 +371,7 @@ wait_for_async_resp(Req_id, Options, Acc_Stat_code, Acc_Headers, Body) ->
maybe_stream_next(Req_id, Options),
wait_for_async_resp(Req_id, Options, StatCode, Headers, Body);
{ibrowse_async_response_end, Req_id} ->
io:format("Recvd end of response.~n", []),
%% io:format("Recvd end of response.~n", []),
Body_1 = list_to_binary(lists:reverse(Body)),
{ok, Acc_Stat_code, Acc_Headers, Body_1};
{ibrowse_async_response, Req_id, Data} ->
@ -384,7 +401,7 @@ execute_req(Url, Method, Options) ->
{ok, SCode, _H, _B} ->
io:format("Status code: ~p~n", [SCode]);
Err ->
io:format("Err -> ~p~n", [Err])
io:format("~p~n", [Err])
end.
drv_ue_test() ->

+ 110
- 0
test/ibrowse_test_server.erl View File

@ -0,0 +1,110 @@
%%% File : ibrowse_test_server.erl
%%% Author : Chandrashekhar Mullaparthi <chandrashekhar.mullaparthi@t-mobile.co.uk>
%%% Description : A server to simulate various test scenarios
%%% Created : 17 Oct 2010 by Chandrashekhar Mullaparthi <chandrashekhar.mullaparthi@t-mobile.co.uk>
-module(ibrowse_test_server).
-export([
start_server/2,
stop_server/1
]).
-record(request, {method, uri, version, headers = [], body = []}).
start_server(Port, Sock_type) ->
Fun = fun() ->
register(server_proc_name(Port), self()),
case do_listen(Sock_type, Port, [{active, false},
{packet, http}]) of
{ok, Sock} ->
do_trace("Server listening on port: ~p~n", [Port]),
accept_loop(Sock, Sock_type);
Err ->
do_trace("Failed to start server on port ~p. ~p~n",
[Port, Err]),
Err
end
end,
spawn(Fun).
stop_server(Port) ->
exit(whereis(server_proc_name(Port)), kill).
server_proc_name(Port) ->
list_to_atom("ibrowse_test_server_"++integer_to_list(Port)).
do_listen(tcp, Port, Opts) ->
gen_tcp:listen(Port, Opts);
do_listen(ssl, Port, Opts) ->
application:start(crypto),
application:start(ssl),
ssl:listen(Port, Opts).
do_accept(tcp, Listen_sock) ->
gen_tcp:accept(Listen_sock);
do_accept(ssl, Listen_sock) ->
ssl:ssl_accept(Listen_sock).
accept_loop(Sock, Sock_type) ->
case do_accept(Sock_type, Sock) of
{ok, Conn} ->
Pid = spawn_link(
fun() ->
server_loop(Conn, Sock_type, #request{})
end),
set_controlling_process(Conn, Sock_type, Pid),
Pid ! {setopts, [{active, true}]},
accept_loop(Sock, Sock_type);
Err ->
Err
end.
set_controlling_process(Sock, tcp, Pid) ->
gen_tcp:controlling_process(Sock, Pid);
set_controlling_process(Sock, ssl, Pid) ->
ssl:controlling_process(Sock, Pid).
setopts(Sock, tcp, Opts) ->
inet:setopts(Sock, Opts);
setopts(Sock, ssl, Opts) ->
ssl:setopts(Sock, Opts).
server_loop(Sock, Sock_type, #request{headers = Headers} = Req) ->
receive
{http, Sock, {http_request, HttpMethod, HttpUri, HttpVersion}} ->
server_loop(Sock, Sock_type, Req#request{method = HttpMethod,
uri = HttpUri,
version = HttpVersion});
{http, Sock, {http_header, _, _, _, _} = H} ->
server_loop(Sock, Sock_type, Req#request{headers = [H | Headers]});
{http, Sock, http_eoh} ->
process_request(Sock, Sock_type, Req),
server_loop(Sock, Sock_type, #request{});
{http, Sock, {http_error, Err}} ->
do_trace("Error parsing HTTP request:~n"
"Req so far : ~p~n"
"Err : ", [Req, Err]),
exit({http_error, Err});
{setopts, Opts} ->
setopts(Sock, Sock_type, Opts),
server_loop(Sock, Sock_type, Req);
Other ->
do_trace("Recvd unknown msg: ~p~n", [Other]),
exit({unknown_msg, Other})
after 5000 ->
do_trace("Timing out client connection~n", []),
ok
end.
do_trace(Fmt, Args) ->
io:format("~s -- " ++ Fmt, [ibrowse_lib:printable_date() | Args]).
process_request(Sock, Sock_type, Req) ->
do_trace("Recvd req: ~p~n", [Req]),
Resp = <<"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n">>,
do_send(Sock, Sock_type, Resp).
do_send(Sock, tcp, Resp) ->
ok = gen_tcp:send(Sock, Resp);
do_send(Sock, ssl, Resp) ->
ok = ssl:send(Sock, Resp).

+ 1
- 1
vsn.mk View File

@ -1,2 +1,2 @@
IBROWSE_VSN = 2.0.1
IBROWSE_VSN = 2.1.0

Loading…
Cancel
Save