diff --git a/Makefile b/Makefile index f0528c3..2380676 100644 --- a/Makefile +++ b/Makefile @@ -9,3 +9,10 @@ clean: install: all mkdir -p $(DESTDIR)/lib/ibrowse-$(IBROWSE_VSN)/ cp -r ebin $(DESTDIR)/lib/ibrowse-$(IBROWSE_VSN)/ + +test: all + erl -noshell -pa ebin -s ibrowse -s ibrowse_test unit_tests \ + -s ibrowse_test verify_chunked_streaming \ + -s ibrowse_test test_chunked_streaming_once \ + -s erlang halt + diff --git a/README b/README index ea948e6..f3ed7a1 100644 --- a/README +++ b/README @@ -22,8 +22,83 @@ Version : 2.0.1 Latest version : git://github.com/cmullaparthi/ibrowse.git +CONTRIBUTORS +============ +The following people have helped maked ibrowse better by reporting bugs, +supplying patches and also asking for new features. Please write to me if you +have contributed and I've missed you out. + +In alphabetical order: + +Adam Kocoloski +Andrew Tunnell-Jones +Anthony Molinaro +Benoit Chesneau +Chris Newcombe +Dan Kelley +Derek Upham +Eric Merritt +Erik Reitsma +Filipe David Manana +Geoff Cant +Jeroen Koops +Joćo Lopes +Karol Skocik +Kostis Sagonas +Matthew Reilly +Oscar Hellström +Paul J. Davis +Peter Kristensen +Ram Krishnan +Richard Cameron +Sean Hinde +Seth Falcon +Steve Vinoski +Thomas Lindgren +YounĆØs Hafri +tholschuh (https://github.com/tholschuh/) + CONTRIBUTIONS & CHANGE HISTORY ============================== +25-10-2010 - v2.1.0 + * Fixed build on OpenSolaris. Bug report and patch from + tholschuh. + http://github.com/cmullaparthi/ibrowse/issues/issue/10 + + * Fixed behaviour of inactivity_timeout option. Reported by + Joćo Lopes. + http://github.com/cmullaparthi/ibrowse/issues/issue/11 + + * Prevent atom table pollution when bogus URLs are input to + ibrowse. Bug report by Joćo Lopes. + http://github.com/cmullaparthi/ibrowse/issues/issue/13 + + * Automatically do Chunked-Transfer encoding of request body + when the body is generated by a fun. Patch provided by + Filipe David Manana. + http://github.com/cmullaparthi/ibrowse/issues/issue/14 + + * Depending on input options, ibrowse sometimes included multiple + Content-Length headers. Bug reported by Paul J. Davis + http://github.com/cmullaparthi/ibrowse/issues/issue/15 + + * Deal with webservers which do not provide a Reason-Phrase on the + response Status-Line. Patch provided by Jeroen Koops. + http://github.com/cmullaparthi/ibrowse/issues/issue/16 + + * Fixed http://github.com/cmullaparthi/ibrowse/issues/issue/17 + This was reported by Filipe David Manana. + + * Fixed http://github.com/cmullaparthi/ibrowse/issues/issue/19 + This was reported by Dan Kelley and Filipe David Manana. + + * Added ibrowse:stream_close/1 to close the connection + associated with a certain response stream. Patch provided by + Joćo Lopes. + + * Prevent port number being included in the Host header when port + 443 is intended. Bug reported by Andrew Tunnell-Jones + 24-09-2010 - v2.0.1 * Removed a spurious io:format statement diff --git a/doc/ibrowse.html b/doc/ibrowse.html index 88cba60..3aec4a8 100644 --- a/doc/ibrowse.html +++ b/doc/ibrowse.html @@ -9,14 +9,14 @@

Module ibrowse

-The ibrowse application implements an HTTP 1.1 client. +The ibrowse application implements an HTTP 1.1 client in erlang.

Copyright © 2005-2010 Chandrashekhar Mullaparthi

-

Version: 2.0.1

+

Version: 2.1.0

Behaviours: gen_server.

Authors: Chandrashekhar Mullaparthi (chandrashekhar dot mullaparthi at gmail dot com).

-

Description

The ibrowse application implements an HTTP 1.1 client. This +

Description

The ibrowse application implements an HTTP 1.1 client in erlang. This module implements the API of the HTTP client. There is one named process called 'ibrowse' which assists in load balancing and maintaining configuration. There is one load balancing process per unique webserver. There is one process to handle one TCP connection to a webserver @@ -87,20 +87,24 @@ send_req/4, send_req/5, send_req/6.

set_dest/3Deprecated. set_max_pipeline_size/3Set the maximum pipeline size for each connection to a specific Host:Port. set_max_sessions/3Set the maximum number of connections allowed to a specific Host:Port. -show_dest_status/0 +show_dest_status/0Shows some internal information about load balancing. show_dest_status/2Shows some internal information about load balancing to a specified Host:Port. spawn_link_worker_process/1Same as spawn_worker_process/1 except the the calling process is linked to the worker process which is spawned. -spawn_link_worker_process/2 +spawn_link_worker_process/2Same as spawn_worker_process/2 except the the calling process + is linked to the worker process which is spawned. spawn_worker_process/1Creates a HTTP client process to the specified Host:Port which is not part of the load balancing pool. -spawn_worker_process/2 +spawn_worker_process/2Same as spawn_worker_process/1 but takes as input a Host and Port + instead of a URL. start/0Starts the ibrowse process without linking. start_link/0Starts the ibrowse process linked to the calling process. stop/0Stop the ibrowse process. stop_worker_process/1Terminate a worker process spawned using spawn_worker_process/2 or spawn_link_worker_process/2. +stream_close/1Tell ibrowse to close the connection associated with the + specified stream. stream_next/1Tell ibrowse to stream the next chunk of data to the caller. terminate/2 @@ -245,11 +249,15 @@ send_req/4, send_req/5, send_req/6.

information as it has, such as HTTP Status Code and HTTP Headers. When this happens, the response is of the form {error, {Reason, {stat_code, StatusCode}, HTTP_headers}} -
  • The inactivity_timeout option is useful when - dealing with large response bodies and/or slow links. In these - cases, it might be hard to estimate how long a request will take to - complete. In such cases, the client might want to timeout if no - data has been received on the link for a certain time interval.
    • +

  • The inactivity_timeout option is useful when +dealing with large response bodies and/or slow links. In these +cases, it might be hard to estimate how long a request will take to +complete. In such cases, the client might want to timeout if no +data has been received on the link for a certain time interval.

    + + This value is also used to close connections which are not in use for + the specified timeout value. +
  • The connect_timeout option is to specify how long the @@ -343,7 +351,9 @@ send_req/4, send_req/5, send_req/6.

    show_dest_status/0

    show_dest_status() -> any()

    -
    +

    Shows some internal information about load balancing. Info + about workers spawned using spawn_worker_process/2 or + spawn_link_worker_process/2 is not included.

    show_dest_status/2

    @@ -362,7 +372,8 @@ send_req/4, send_req/5, send_req/6.

    spawn_link_worker_process/2

    spawn_link_worker_process(Host::string(), Port::integer()) -> {ok, pid()}

    -
    +

    Same as spawn_worker_process/2 except the the calling process + is linked to the worker process which is spawned.

    spawn_worker_process/1

    @@ -383,7 +394,8 @@ send_req/4, send_req/5, send_req/6.

    spawn_worker_process/2

    spawn_worker_process(Host::string(), Port::integer()) -> {ok, pid()}

    -
    +

    Same as spawn_worker_process/1 but takes as input a Host and Port + instead of a URL.

    start/0

    @@ -407,6 +419,15 @@ send_req/4, send_req/5, send_req/6.

    spawn_worker_process/2 or spawn_link_worker_process/2. Requests in progress will get the error response 
    {error, closing_on_request}

    +

    stream_close/1

    +
    +

    stream_close(Req_id::req_id()) -> ok | {error, unknown_req_id}

    +

    Tell ibrowse to close the connection associated with the + specified stream. Should be used in conjunction with the + stream_to option. Note that all requests in progress on + the connection which is serving this Req_id will be aborted, and an + error returned.

    +

    stream_next/1

    stream_next(Req_id::req_id()) -> ok | {error, unknown_req_id}

    @@ -446,6 +467,6 @@ send_req/4, send_req/5, send_req/6.

    Overviewerlang logo
    -

    Generated by EDoc, Sep 24 2010, 06:42:36.

    +

    Generated by EDoc, Nov 10 2010, 06:04:33.

    diff --git a/ebin/ibrowse.app b/ebin/ibrowse.app index b72099d..e8580d1 100644 --- a/ebin/ibrowse.app +++ b/ebin/ibrowse.app @@ -1,6 +1,6 @@ {application, ibrowse, [{description, "HTTP client application"}, - {vsn, "2.0.0"}, + {vsn, "2.1.0"}, {modules, [ ibrowse, ibrowse_http_client, ibrowse_app, diff --git a/src/Makefile b/src/Makefile index 370ede4..145c0a2 100644 --- a/src/Makefile +++ b/src/Makefile @@ -24,7 +24,7 @@ $(EBIN)/%.beam: %.erl ${ERLC} $(COMPILER_OPTIONS) $(INCLUDE_DIRS) -o ../ebin $< $(EBIN)/%.app: %.app.src ../vsn.mk Makefile - sed -e s^%IBROWSE_VSN%^$(IBROWSE_VSN)^ \ + sed -e s/%IBROWSE_VSN%/$(IBROWSE_VSN)/ \ $< > $@ clean: diff --git a/src/ibrowse.erl b/src/ibrowse.erl index 7f8d8bc..1a42f4b 100644 --- a/src/ibrowse.erl +++ b/src/ibrowse.erl @@ -7,8 +7,8 @@ %%%------------------------------------------------------------------- %% @author Chandrashekhar Mullaparthi %% @copyright 2005-2010 Chandrashekhar Mullaparthi -%% @version 2.0.1 -%% @doc The ibrowse application implements an HTTP 1.1 client. This +%% @version 2.1.0 +%% @doc The ibrowse application implements an HTTP 1.1 client in erlang. This %% module implements the API of the HTTP client. There is one named %% process called 'ibrowse' which assists in load balancing and maintaining configuration. There is one load balancing process per unique webserver. There is %% one process to handle one TCP connection to a webserver @@ -87,6 +87,7 @@ send_req_direct/6, send_req_direct/7, stream_next/1, + stream_close/1, set_max_sessions/3, set_max_pipeline_size/3, set_dest/3, @@ -201,7 +202,11 @@ send_req(Url, Headers, Method, Body) -> %% dealing with large response bodies and/or slow links. In these %% cases, it might be hard to estimate how long a request will take to %% complete. In such cases, the client might want to timeout if no -%% data has been received on the link for a certain time interval.
    • +%% data has been received on the link for a certain time interval. +%% +%% This value is also used to close connections which are not in use for +%% the specified timeout value. +%% %% %%
  • %% The connect_timeout option is to specify how long the @@ -458,6 +463,8 @@ ensure_bin({Fun, _} = Body) when is_function(Fun) -> Body. spawn_worker_process(Url) -> ibrowse_http_client:start(Url). +%% @doc Same as spawn_worker_process/1 but takes as input a Host and Port +%% instead of a URL. %% @spec spawn_worker_process(Host::string(), Port::integer()) -> {ok, pid()} spawn_worker_process(Host, Port) -> ibrowse_http_client:start({Host, Port}). @@ -468,6 +475,8 @@ spawn_worker_process(Host, Port) -> spawn_link_worker_process(Url) -> ibrowse_http_client:start_link(Url). +%% @doc Same as spawn_worker_process/2 except the the calling process +%% is linked to the worker process which is spawned. %% @spec spawn_link_worker_process(Host::string(), Port::integer()) -> {ok, pid()} spawn_link_worker_process(Host, Port) -> ibrowse_http_client:start_link({Host, Port}). @@ -524,6 +533,21 @@ stream_next(Req_id) -> ok end. +%% @doc Tell ibrowse to close the connection associated with the +%% specified stream. Should be used in conjunction with the +%% stream_to option. Note that all requests in progress on +%% the connection which is serving this Req_id will be aborted, and an +%% error returned. +%% @spec stream_close(Req_id :: req_id()) -> ok | {error, unknown_req_id} +stream_close(Req_id) -> + case ets:lookup(ibrowse_stream, {req_id_pid, Req_id}) of + [] -> + {error, unknown_req_id}; + [{_, Pid}] -> + catch Pid ! {stream_close, Req_id}, + ok + end. + %% @doc Turn tracing on for the ibrowse process trace_on() -> ibrowse ! {trace, true}. @@ -553,6 +577,9 @@ all_trace_off() -> ibrowse ! all_trace_off, ok. +%% @doc Shows some internal information about load balancing. Info +%% about workers spawned using spawn_worker_process/2 or +%% spawn_link_worker_process/2 is not included. show_dest_status() -> Dests = lists:filter(fun({lb_pid, {Host, Port}, _}) when is_list(Host), is_integer(Port) -> diff --git a/src/ibrowse_http_client.erl b/src/ibrowse_http_client.erl index 16d9b87..5c3d5c9 100644 --- a/src/ibrowse_http_client.erl +++ b/src/ibrowse_http_client.erl @@ -37,6 +37,7 @@ -include("ibrowse.hrl"). -record(state, {host, port, connect_timeout, + inactivity_timer_ref, use_proxy = false, proxy_auth_digest, ssl_options = [], is_ssl = false, socket, proxy_tunnel_setup = false, @@ -192,6 +193,12 @@ handle_info({stream_next, Req_id}, #state{socket = Socket, handle_info({stream_next, _Req_id}, State) -> {noreply, State}; +handle_info({stream_close, _Req_id}, State) -> + shutting_down(State), + do_close(State), + do_error_reply(State, closing_on_request), + {stop, normal, ok, State}; + handle_info({tcp_closed, _Sock}, State) -> do_trace("TCP connection closed by peer!~n", []), handle_sock_closed(State), @@ -221,6 +228,7 @@ handle_info({req_timedout, From}, State) -> end; handle_info(timeout, State) -> + do_trace("Inactivity timeout triggered. Shutting down connection~n", []), shutting_down(State), do_error_reply(State, req_timedout), {stop, normal, State}; @@ -273,8 +281,8 @@ handle_sock_data(Data, #state{status = get_header}=State) -> {stop, normal, State}; State_1 -> active_once(State_1), - set_inac_timer(State_1), - {noreply, State_1} + State_2 = set_inac_timer(State_1), + {noreply, State_2} end; handle_sock_data(Data, #state{status = get_body, @@ -293,8 +301,8 @@ handle_sock_data(Data, #state{status = get_body, {stop, normal, State}; State_1 -> active_once(State_1), - set_inac_timer(State_1), - {noreply, State_1} + State_2 = set_inac_timer(State_1), + {noreply, State_2} end; _ -> case parse_11_response(Data, State) of @@ -314,12 +322,12 @@ handle_sock_data(Data, #state{status = get_body, active_once(State_1) end, State_2 = State_1#state{interim_reply_sent = false}, - set_inac_timer(State_2), - {noreply, State_2}; + State_3 = set_inac_timer(State_2), + {noreply, State_3}; State_1 -> active_once(State_1), - set_inac_timer(State_1), - {noreply, State_1} + State_2 = set_inac_timer(State_1), + {noreply, State_2} end end. @@ -507,29 +515,37 @@ do_send(Req, #state{socket = Sock, is_ssl = false}) -> gen_tcp:send(Sock, Req). %% {fun_arity_0} | %% {fun_arity_1, term()} %% error() = term() -do_send_body(Source, State) when is_function(Source) -> - do_send_body({Source}, State); -do_send_body({Source}, State) when is_function(Source) -> - do_send_body1(Source, Source(), State); -do_send_body({Source, Source_state}, State) when is_function(Source) -> - do_send_body1(Source, Source(Source_state), State); -do_send_body(Body, State) -> +do_send_body(Source, State, TE) when is_function(Source) -> + do_send_body({Source}, State, TE); +do_send_body({Source}, State, TE) when is_function(Source) -> + do_send_body1(Source, Source(), State, TE); +do_send_body({Source, Source_state}, State, TE) when is_function(Source) -> + do_send_body1(Source, Source(Source_state), State, TE); +do_send_body(Body, State, _TE) -> do_send(Body, State). -do_send_body1(Source, Resp, State) -> +do_send_body1(Source, Resp, State, TE) -> case Resp of {ok, Data} -> - do_send(Data, State), - do_send_body({Source}, State); + do_send(maybe_chunked_encode(Data, TE), State), + do_send_body({Source}, State, TE); {ok, Data, New_source_state} -> - do_send(Data, State), - do_send_body({Source, New_source_state}, State); + do_send(maybe_chunked_encode(Data, TE), State), + do_send_body({Source, New_source_state}, State, TE); + eof when TE == true -> + do_send(<<"0\r\n\r\n">>, State), + ok; eof -> ok; Err -> Err end. +maybe_chunked_encode(Data, false) -> + Data; +maybe_chunked_encode(Data, true) -> + [ibrowse_lib:dec2hex(4, size(to_binary(Data))), "\r\n", Data, "\r\n"]. + do_close(#state{socket = undefined}) -> ok; do_close(#state{socket = Sock, is_ssl = true, @@ -619,11 +635,13 @@ send_req_1(From, {Req, Body_1} = make_request(connect, Pxy_auth_headers, Path, Path, [], Options, State_1), + TE = is_chunked_encoding_specified(Options), trace_request(Req), case do_send(Req, State) of ok -> - case do_send_body(Body_1, State_1) of + case do_send_body(Body_1, State_1, TE) of ok -> + trace_request_body(Body_1), active_once(State_1), Ref = case Timeout of infinity -> @@ -636,8 +654,8 @@ send_req_1(From, send_timer = Ref, proxy_tunnel_setup = in_progress, tunnel_setup_queue = [{From, Url, Headers, Method, Body, Options, Timeout}]}, - set_inac_timer(State_1), - {noreply, State_2}; + State_3 = set_inac_timer(State_2), + {noreply, State_3}; Err -> shutting_down(State_1), do_trace("Send failed... Reason: ~p~n", [Err]), @@ -706,10 +724,12 @@ send_req_1(From, AbsPath, RelPath, Body, Options, State_1), trace_request(Req), do_setopts(Socket, Caller_socket_options, Is_ssl), + TE = is_chunked_encoding_specified(Options), case do_send(Req, State_1) of ok -> - case do_send_body(Body_1, State_1) of + case do_send_body(Body_1, State_1, TE) of ok -> + trace_request_body(Body_1), State_2 = inc_pipeline_counter(State_1), active_once(State_2), Ref = case Timeout of @@ -732,8 +752,8 @@ send_req_1(From, _ -> gen_server:reply(From, {ibrowse_req_id, ReqId}) end, - set_inac_timer(State_1), - {noreply, State_3}; + State_4 = set_inac_timer(State_3), + {noreply, State_4}; Err -> shutting_down(State_1), do_trace("Send failed... Reason: ~p~n", [Err]), @@ -759,6 +779,7 @@ maybe_modify_headers(#url{host = Host, port = Port} = Url, false -> case Port of 80 -> Host; + 443 -> Host; _ -> [Host, ":", integer_to_list(Port)] end; {value, {_, Host_h_val}} -> @@ -802,31 +823,42 @@ http_auth_digest(Username, Password) -> make_request(Method, Headers, AbsPath, RelPath, Body, Options, #state{use_proxy = UseProxy, is_ssl = Is_ssl}) -> HttpVsn = http_vsn_string(get_value(http_vsn, Options, {1,1})), + Fun1 = fun({X, Y}) when is_atom(X) -> + {to_lower(atom_to_list(X)), X, Y}; + ({X, Y}) when is_list(X) -> + {to_lower(X), X, Y} + end, + Headers_0 = [Fun1(X) || X <- Headers], Headers_1 = - case get_value(content_length, Headers, false) of - false when (Body == []) or - (Body == <<>>) or - is_tuple(Body) or - is_function(Body) -> - Headers; + case lists:keysearch("content-length", 1, Headers_0) of + false when (Body == []) orelse + (Body == <<>>) orelse + is_tuple(Body) orelse + is_function(Body) -> + Headers_0; false when is_binary(Body) -> - [{"content-length", integer_to_list(size(Body))} | Headers]; - false -> - [{"content-length", integer_to_list(length(Body))} | Headers]; + [{"content-length", "content-length", integer_to_list(size(Body))} | Headers_0]; + false when is_list(Body) -> + [{"content-length", "content-length", integer_to_list(length(Body))} | Headers_0]; _ -> - Headers + %% Content-Length is already specified + Headers_0 end, {Headers_2, Body_1} = - case get_value(transfer_encoding, Options, false) of + case is_chunked_encoding_specified(Options) of false -> - {Headers_1, Body}; - {chunked, ChunkSize} -> - {[{X, Y} || {X, Y} <- Headers_1, - X /= "Content-Length", - X /= "content-length", - X /= content_length] ++ + {[{Y, Z} || {_, Y, Z} <- Headers_1], Body}; + true -> + Chunk_size_1 = case get_value(transfer_encoding, Options) of + chunked -> + 5120; + {chunked, Chunk_size} -> + Chunk_size + end, + {[{Y, Z} || {X, Y, Z} <- Headers_1, + X /= "content-length"] ++ [{"Transfer-Encoding", "chunked"}], - chunk_request_body(Body, ChunkSize)} + chunk_request_body(Body, Chunk_size_1)} end, Headers_3 = cons_headers(Headers_2), Uri = case get_value(use_absolute_uri, Options, false) or UseProxy of @@ -842,6 +874,16 @@ make_request(Method, Headers, AbsPath, RelPath, Body, Options, end, {[method(Method), " ", Uri, " ", HttpVsn, crnl(), Headers_3, crnl()], Body_1}. +is_chunked_encoding_specified(Options) -> + case get_value(transfer_encoding, Options, false) of + false -> + false; + {chunked, _} -> + true; + chunked -> + true + end. + http_vsn_string({0,9}) -> "HTTP/0.9"; http_vsn_string({1,0}) -> "HTTP/1.0"; http_vsn_string({1,1}) -> "HTTP/1.1". @@ -873,6 +915,9 @@ encode_headers([{Name,Val} | T], Acc) when is_atom(Name) -> encode_headers([], Acc) -> lists:reverse(Acc). +chunk_request_body(Body, _ChunkSize) when is_tuple(Body) orelse + is_function(Body) -> + Body; chunk_request_body(Body, ChunkSize) -> chunk_request_body(Body, ChunkSize, []). @@ -1060,7 +1105,7 @@ upgrade_to_ssl(#state{socket = Socket, send_queued_requests([], State) -> do_trace("Sent all queued requests via SSL connection~n", []), - State#state{tunnel_setup_queue = done}; + State#state{tunnel_setup_queue = []}; send_queued_requests([{From, Url, Headers, Method, Body, Options, Timeout} | Q], State) -> case send_req_1(From, Url, Headers, Method, Body, Options, Timeout, State) of @@ -1217,7 +1262,6 @@ handle_response(#request{from=From, stream_to=StreamTo, req_id=ReqId, reply_buffer = RepBuf, recvd_headers = RespHeaders}=State) when SaveResponseToFile /= false -> Body = RepBuf, - State_1 = set_cur_request(State), file:close(Fd), ResponseBody = case TmpFilename of undefined -> @@ -1232,9 +1276,9 @@ handle_response(#request{from=From, stream_to=StreamTo, req_id=ReqId, false -> {ok, SCode, Resp_headers_1, ResponseBody} end, - State_2 = do_reply(State_1, From, StreamTo, ReqId, Resp_format, Reply), + State_1 = do_reply(State, From, StreamTo, ReqId, Resp_format, Reply), cancel_timer(ReqTimer, {eat_message, {req_timedout, From}}), - State_2; + set_cur_request(State_1); handle_response(#request{from=From, stream_to=StreamTo, req_id=ReqId, response_format = Resp_format, options = Options}, @@ -1245,7 +1289,6 @@ handle_response(#request{from=From, stream_to=StreamTo, req_id=ReqId, reply_buffer = RepBuf, send_timer = ReqTimer} = State) -> Body = RepBuf, -%% State_1 = set_cur_request(State), {Resp_headers_1, Raw_headers_1} = maybe_add_custom_headers(Resp_headers, Raw_headers, Options), Reply = case get_value(give_raw_headers, Options, false) of true -> @@ -1253,15 +1296,8 @@ handle_response(#request{from=From, stream_to=StreamTo, req_id=ReqId, false -> {ok, SCode, Resp_headers_1, Body} end, - State_1 = case get(conn_close) of - "close" -> - do_reply(State, From, StreamTo, ReqId, Resp_format, Reply), - exit(normal); - _ -> - State_1_1 = do_reply(State, From, StreamTo, ReqId, Resp_format, Reply), - cancel_timer(ReqTimer, {eat_message, {req_timedout, From}}), - State_1_1 - end, + State_1 = do_reply(State, From, StreamTo, ReqId, Resp_format, Reply), + cancel_timer(ReqTimer, {eat_message, {req_timedout, From}}), set_cur_request(State_1). reset_state(State) -> @@ -1353,6 +1389,8 @@ parse_status_line([32 | T], get_prot_vsn, ProtVsn, StatCode) -> parse_status_line(T, get_status_code, ProtVsn, StatCode); parse_status_line([32 | T], get_status_code, ProtVsn, StatCode) -> {ok, lists:reverse(ProtVsn), lists:reverse(StatCode), T}; +parse_status_line([], get_status_code, ProtVsn, StatCode) -> + {ok, lists:reverse(ProtVsn), lists:reverse(StatCode), []}; parse_status_line([H | T], get_prot_vsn, ProtVsn, StatCode) -> parse_status_line(T, get_prot_vsn, [H|ProtVsn], StatCode); parse_status_line([H | T], get_status_code, ProtVsn, StatCode) -> @@ -1710,28 +1748,61 @@ get_stream_chunk_size(Options) -> end. set_inac_timer(State) -> - set_inac_timer(State, get_inac_timeout(State)). + cancel_timer(State#state.inactivity_timer_ref), + set_inac_timer(State#state{inactivity_timer_ref = undefined}, + get_inac_timeout(State)). -set_inac_timer(_State, Timeout) when is_integer(Timeout) -> - erlang:send_after(Timeout, self(), timeout); -set_inac_timer(_, _) -> - undefined. +set_inac_timer(State, Timeout) when is_integer(Timeout) -> + Ref = erlang:send_after(Timeout, self(), timeout), + State#state{inactivity_timer_ref = Ref}; +set_inac_timer(State, _) -> + State. get_inac_timeout(#state{cur_req = #request{options = Opts}}) -> get_value(inactivity_timeout, Opts, infinity); get_inac_timeout(#state{cur_req = undefined}) -> - infinity. + case ibrowse:get_config_value(inactivity_timeout, undefined) of + Val when is_integer(Val) -> + Val; + _ -> + case application:get_env(ibrowse, inactivity_timeout) of + {ok, Val} when is_integer(Val), Val > 0 -> + Val; + _ -> + 10000 + end + end. trace_request(Req) -> case get(my_trace_flag) of true -> %%Avoid the binary operations if trace is not on... - NReq = binary_to_list(list_to_binary(Req)), + NReq = to_binary(Req), do_trace("Sending request: ~n" "--- Request Begin ---~n~s~n" "--- Request End ---~n", [NReq]); _ -> ok end. +trace_request_body(Body) -> + case get(my_trace_flag) of + true -> + %%Avoid the binary operations if trace is not on... + NBody = to_binary(Body), + case size(NBody) > 1024 of + true -> + ok; + false -> + do_trace("Sending request body: ~n" + "--- Request Body Begin ---~n~s~n" + "--- Request Body End ---~n", [NBody]) + end; + false -> + ok + end. + to_integer(X) when is_list(X) -> list_to_integer(X); to_integer(X) when is_integer(X) -> X. + +to_binary(X) when is_list(X) -> list_to_binary(X); +to_binary(X) when is_binary(X) -> X. diff --git a/src/ibrowse_lib.erl b/src/ibrowse_lib.erl index fbb9c34..c463c7b 100644 --- a/src/ibrowse_lib.erl +++ b/src/ibrowse_lib.erl @@ -208,7 +208,7 @@ parse_url(Url) -> parse_url([$:, $/, $/ | _], get_protocol, Url, []) -> {invalid_uri_1, Url}; parse_url([$:, $/, $/ | T], get_protocol, Url, TmpAcc) -> - Prot = list_to_atom(lists:reverse(TmpAcc)), + Prot = list_to_existing_atom(lists:reverse(TmpAcc)), parse_url(T, get_username, Url#url{protocol = Prot}, []); diff --git a/src/ibrowse_test.erl b/src/ibrowse_test.erl index e7d6e59..3ad7660 100644 --- a/src/ibrowse_test.erl +++ b/src/ibrowse_test.erl @@ -217,14 +217,18 @@ dump_errors(Key, Iod) -> {"http://jigsaw.w3.org/HTTP/300/", get}, {"http://jigsaw.w3.org/HTTP/Basic/", get, [{basic_auth, {"guest", "guest"}}]}, {"http://jigsaw.w3.org/HTTP/CL/", get}, - {"http://www.httpwatch.com/httpgallery/chunked/", get} + {"http://www.httpwatch.com/httpgallery/chunked/", get}, + {"https://github.com", get, [{ssl_options, [{depth, 2}]}]} ]). unit_tests() -> unit_tests([]). unit_tests(Options) -> + application:start(crypto), + application:start(public_key), application:start(ssl), + ibrowse:start(), Options_1 = Options ++ [{connect_timeout, 5000}], {Pid, Ref} = erlang:spawn_monitor(?MODULE, unit_tests_1, [self(), Options_1]), receive @@ -249,32 +253,45 @@ verify_chunked_streaming() -> verify_chunked_streaming([]). verify_chunked_streaming(Options) -> + io:format("~nVerifying that chunked streaming is working...~n", []), Url = "http://www.httpwatch.com/httpgallery/chunked/", - io:format("URL: ~s~n", [Url]), - io:format("Fetching data without streaming...~n", []), + io:format(" URL: ~s~n", [Url]), + io:format(" Fetching data without streaming...~n", []), Result_without_streaming = ibrowse:send_req( Url, [], get, [], [{response_format, binary} | Options]), - io:format("Fetching data with streaming as list...~n", []), + io:format(" Fetching data with streaming as list...~n", []), Async_response_list = do_async_req_list( Url, get, [{response_format, list} | Options]), - io:format("Fetching data with streaming as binary...~n", []), + io:format(" Fetching data with streaming as binary...~n", []), Async_response_bin = do_async_req_list( Url, get, [{response_format, binary} | Options]), - io:format("Fetching data with streaming as binary, {active, once}...~n", []), + io:format(" Fetching data with streaming as binary, {active, once}...~n", []), Async_response_bin_once = do_async_req_list( Url, get, [once, {response_format, binary} | Options]), - compare_responses(Result_without_streaming, Async_response_list, Async_response_bin), - compare_responses(Result_without_streaming, Async_response_list, Async_response_bin_once). + Res1 = compare_responses(Result_without_streaming, Async_response_list, Async_response_bin), + Res2 = compare_responses(Result_without_streaming, Async_response_list, Async_response_bin_once), + case {Res1, Res2} of + {success, success} -> + io:format(" Chunked streaming working~n", []); + _ -> + ok + end. test_chunked_streaming_once() -> test_chunked_streaming_once([]). test_chunked_streaming_once(Options) -> + io:format("~nTesting chunked streaming with the {stream_to, {Pid, once}} option...~n", []), Url = "http://www.httpwatch.com/httpgallery/chunked/", - io:format("URL: ~s~n", [Url]), - io:format("Fetching data with streaming as binary, {active, once}...~n", []), - do_async_req_list(Url, get, [once, {response_format, binary} | Options]). + io:format(" URL: ~s~n", [Url]), + io:format(" Fetching data with streaming as binary, {active, once}...~n", []), + case do_async_req_list(Url, get, [once, {response_format, binary} | Options]) of + {ok, _, _, _} -> + io:format(" Success!~n", []); + Err -> + io:format(" Fail: ~p~n", [Err]) + end. compare_responses({ok, St_code, _, Body}, {ok, St_code, _, Body}, {ok, St_code, _, Body}) -> success; @@ -310,7 +327,7 @@ do_async_req_list(Url, Method, Options) -> {Pid,_} = erlang:spawn_monitor(?MODULE, i_do_async_req_list, [self(), Url, Method, Options ++ [{stream_chunk_size, 1000}]]), - io:format("Spawned process ~p~n", [Pid]), +%% io:format("Spawned process ~p~n", [Pid]), wait_for_resp(Pid). wait_for_resp(Pid) -> @@ -354,7 +371,7 @@ wait_for_async_resp(Req_id, Options, Acc_Stat_code, Acc_Headers, Body) -> maybe_stream_next(Req_id, Options), wait_for_async_resp(Req_id, Options, StatCode, Headers, Body); {ibrowse_async_response_end, Req_id} -> - io:format("Recvd end of response.~n", []), + %% io:format("Recvd end of response.~n", []), Body_1 = list_to_binary(lists:reverse(Body)), {ok, Acc_Stat_code, Acc_Headers, Body_1}; {ibrowse_async_response, Req_id, Data} -> @@ -384,7 +401,7 @@ execute_req(Url, Method, Options) -> {ok, SCode, _H, _B} -> io:format("Status code: ~p~n", [SCode]); Err -> - io:format("Err -> ~p~n", [Err]) + io:format("~p~n", [Err]) end. drv_ue_test() -> diff --git a/test/ibrowse_test_server.erl b/test/ibrowse_test_server.erl new file mode 100644 index 0000000..37d760f --- /dev/null +++ b/test/ibrowse_test_server.erl @@ -0,0 +1,110 @@ +%%% File : ibrowse_test_server.erl +%%% Author : Chandrashekhar Mullaparthi +%%% Description : A server to simulate various test scenarios +%%% Created : 17 Oct 2010 by Chandrashekhar Mullaparthi + +-module(ibrowse_test_server). +-export([ + start_server/2, + stop_server/1 + ]). + +-record(request, {method, uri, version, headers = [], body = []}). + +start_server(Port, Sock_type) -> + Fun = fun() -> + register(server_proc_name(Port), self()), + case do_listen(Sock_type, Port, [{active, false}, + {packet, http}]) of + {ok, Sock} -> + do_trace("Server listening on port: ~p~n", [Port]), + accept_loop(Sock, Sock_type); + Err -> + do_trace("Failed to start server on port ~p. ~p~n", + [Port, Err]), + Err + end + end, + spawn(Fun). + +stop_server(Port) -> + exit(whereis(server_proc_name(Port)), kill). + +server_proc_name(Port) -> + list_to_atom("ibrowse_test_server_"++integer_to_list(Port)). + +do_listen(tcp, Port, Opts) -> + gen_tcp:listen(Port, Opts); +do_listen(ssl, Port, Opts) -> + application:start(crypto), + application:start(ssl), + ssl:listen(Port, Opts). + +do_accept(tcp, Listen_sock) -> + gen_tcp:accept(Listen_sock); +do_accept(ssl, Listen_sock) -> + ssl:ssl_accept(Listen_sock). + +accept_loop(Sock, Sock_type) -> + case do_accept(Sock_type, Sock) of + {ok, Conn} -> + Pid = spawn_link( + fun() -> + server_loop(Conn, Sock_type, #request{}) + end), + set_controlling_process(Conn, Sock_type, Pid), + Pid ! {setopts, [{active, true}]}, + accept_loop(Sock, Sock_type); + Err -> + Err + end. + +set_controlling_process(Sock, tcp, Pid) -> + gen_tcp:controlling_process(Sock, Pid); +set_controlling_process(Sock, ssl, Pid) -> + ssl:controlling_process(Sock, Pid). + +setopts(Sock, tcp, Opts) -> + inet:setopts(Sock, Opts); +setopts(Sock, ssl, Opts) -> + ssl:setopts(Sock, Opts). + +server_loop(Sock, Sock_type, #request{headers = Headers} = Req) -> + receive + {http, Sock, {http_request, HttpMethod, HttpUri, HttpVersion}} -> + server_loop(Sock, Sock_type, Req#request{method = HttpMethod, + uri = HttpUri, + version = HttpVersion}); + {http, Sock, {http_header, _, _, _, _} = H} -> + server_loop(Sock, Sock_type, Req#request{headers = [H | Headers]}); + {http, Sock, http_eoh} -> + process_request(Sock, Sock_type, Req), + server_loop(Sock, Sock_type, #request{}); + {http, Sock, {http_error, Err}} -> + do_trace("Error parsing HTTP request:~n" + "Req so far : ~p~n" + "Err : ", [Req, Err]), + exit({http_error, Err}); + {setopts, Opts} -> + setopts(Sock, Sock_type, Opts), + server_loop(Sock, Sock_type, Req); + Other -> + do_trace("Recvd unknown msg: ~p~n", [Other]), + exit({unknown_msg, Other}) + after 5000 -> + do_trace("Timing out client connection~n", []), + ok + end. + +do_trace(Fmt, Args) -> + io:format("~s -- " ++ Fmt, [ibrowse_lib:printable_date() | Args]). + +process_request(Sock, Sock_type, Req) -> + do_trace("Recvd req: ~p~n", [Req]), + Resp = <<"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n">>, + do_send(Sock, Sock_type, Resp). + +do_send(Sock, tcp, Resp) -> + ok = gen_tcp:send(Sock, Resp); +do_send(Sock, ssl, Resp) -> + ok = ssl:send(Sock, Resp). diff --git a/vsn.mk b/vsn.mk index d13db2d..9e6b36d 100644 --- a/vsn.mk +++ b/vsn.mk @@ -1,2 +1,2 @@ -IBROWSE_VSN = 2.0.1 +IBROWSE_VSN = 2.1.0