Version 4.1.0

.gitignore

@@ -7,4 +7,5 @@ doc/*.css
 doc/*.png
 doc/edoc-info
 Emakefile
-*.bat
+*.bat
+.dialyzer_plt

.travis.yml

@@ -1,9 +1,4 @@
 language: erlang
 otp_release:
-   - R14B04
-   - R14B03
-   - R14B02
-   - R14A
-   - R13B04
-   - R13B03
+   - R16B
 script: "make test"

BSD_LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2005-2013, Chandrashekhar Mullaparthi
+Copyright (c) 2005-2014, Chandrashekhar Mullaparthi
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:

CHANGELOG

@@ -1,5 +1,12 @@
 CONTRIBUTIONS & CHANGE HISTORY
 ==============================
+18-04-2013 - v4.1.0
+             * Fix for https://github.com/cmullaparthi/ibrowse/issues/101
+             * Support for https://github.com/cmullaparthi/ibrowse/issues/90
+             * Fix for https://github.com/cmullaparthi/ibrowse/issues/86
+             * Merged various contributions. Please see commit history for details
+             * Introduced the return_raw_request option
+
 09-04-2013 - v4.0.2
              * Tagging master with new version to cover changes
                contributed over the past few months via various pull requests

CONTRIBUTORS

@@ -42,11 +42,16 @@ fholzhauser (https://github.com/fholzhauser/)
 hyperthunk (https://github.com/hyperthunk/)
 Mistagrooves (https://github.com/Mistagrooves/)
 tholschuh (https://github.com/tholschuh/)
-https://github.com/nrdufour
+https://github.com/apauley
+https://github.com/AeroNotix
+https://github.com/dis
+https://github.com/f355
+https://github.com/flycodepl
 https://github.com/helllamer
-https://github.com/rflynn
-https://github.com/puzza007
 https://github.com/marutha
-https://github.com/dis
+https://github.com/nrdufour
 https://github.com/pib
+https://github.com/puzza007
+https://github.com/rflynn
+https://github.com/Vagabond
 

LICENSE

@@ -1,5 +1,5 @@
 ibrowse - a HTTP client written in erlang
-Copyright (C) 2005-2013 Chandrashekhar Mullaparthi <chandrashekhar dot mullaparthi at gmail dot com>
+Copyright (C) 2005-2014 Chandrashekhar Mullaparthi <chandrashekhar dot mullaparthi at gmail dot com>
 
 This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation; either version 2.1 of the License, or (at your option) any later version.
 

Makefile

@@ -1,12 +1,17 @@
 IBROWSE_VSN = $(shell sed -n 's/.*{vsn,.*"\(.*\)"}.*/\1/p' src/ibrowse.app.src)
 
-all:
+DIALYZER_PLT=$(CURDIR)/.dialyzer_plt
+DIALYZER_APPS=erts kernel stdlib ssl crypto public_key
+
+all: compile
+
+compile:
 	./rebar compile
 
 clean:
 	./rebar clean
 
-install: all
+install: compile
 	mkdir -p $(DESTDIR)/lib/ibrowse-$(IBROWSE_VSN)/
 	cp -r ebin $(DESTDIR)/lib/ibrowse-$(IBROWSE_VSN)/
 
@@ -23,3 +28,15 @@ xref: all
 docs:
 	erl -noshell \
 		-eval 'edoc:application(ibrowse, ".", []), init:stop().'
+
+$(DIALYZER_PLT):
+	@echo Creating dialyzer plt file: $(DIALYZER_PLT)
+	@echo This may take a minute or two...
+	@echo
+	dialyzer --output_plt $(DIALYZER_PLT) --build_plt \
+	   --apps $(DIALYZER_APPS)
+
+dialyzer: $(DIALYZER_PLT)
+	@echo Running dialyzer...
+	@echo
+	dialyzer --fullpath --plt $(DIALYZER_PLT) -Wrace_conditions -Wunmatched_returns -Werror_handling -r ./ebin

README.md

@@ -7,7 +7,7 @@ ibrowse is a HTTP client written in erlang.
 
 **Comments to:** chandrashekhar.mullaparthi@gmail.com
 
-**Current Version:** 4.0.2
+**Current Version:** 4.1.0
 
 **Latest Version:** git://github.com/cmullaparthi/ibrowse.git
 

rebar.config

@@ -1,3 +1,3 @@
-{erl_opts, [debug_info, warn_unused_vars, nowarn_shadow_vars, warn_unused_import]}.
+{erl_opts, [debug_info, warnings_as_errors, warn_unused_vars, nowarn_shadow_vars, warn_unused_import]}.
 {xref_checks, [undefined_function_calls, deprecated_function_calls]}.
 {eunit_opts, [verbose]}.

src/ibrowse.erl

@@ -6,7 +6,7 @@
 %%% Created : 11 Oct 2003 by Chandrashekhar Mullaparthi <chandrashekhar.mullaparthi@t-mobile.co.uk>
 %%%-------------------------------------------------------------------
 %% @author Chandrashekhar Mullaparthi <chandrashekhar dot mullaparthi at gmail dot com>
-%% @copyright 2005-2012 Chandrashekhar Mullaparthi
+%% @copyright 2005-2014 Chandrashekhar Mullaparthi
 %% @doc The ibrowse application implements an HTTP 1.1 client in erlang. This
 %% module implements the API of the HTTP client. There is one named
 %% process called 'ibrowse' which assists in load balancing and maintaining configuration. There is one load balancing process per unique webserver. There is
@@ -158,7 +158,7 @@ stop() ->
 %% respHeader() = {headerName(), headerValue()}
 %% headerName() = string()
 %% headerValue() = string()
-%% response() = {ok, Status, ResponseHeaders, ResponseBody} | {ibrowse_req_id, req_id() } | {error, Reason}
+%% response() = {ok, Status, ResponseHeaders, ResponseBody} | {ok, Status, ResponseHeaders, ResponseBody} | {ibrowse_req_id, req_id() } | {error, Reason}
 %% req_id() = term()
 %% ResponseBody = string() | {file, Filename}
 %% Reason = term()
@@ -256,6 +256,8 @@ send_req(Url, Headers, Method, Body) ->
 %% to receive the raw data stream when the Transfer-Encoding of the server
 %% response is Chunked.
 %% </li>
+%% <li> The <code>return_raw_request</code> option enables the caller to get the exact request which was sent by ibrowse to the server, along with the response. When this option is used, the response for synchronous requests is a 5-tuple instead of the usual 4-tuple. For asynchronous requests, the calling process gets a message <code>{ibrowse_async_raw_req, Raw_req}</code>. 
+%% </li>
 %% </ul>
 %%
 %% @spec send_req(Url::string(), Headers::headerList(), Method::method(), Body::body(), Options::optionList()) -> response()

src/ibrowse_http_client.erl

@@ -194,7 +194,7 @@ handle_info({ssl, _Sock, Data}, State) ->
 
 handle_info({stream_next, Req_id}, #state{socket = Socket,
                                           cur_req = #request{req_id = Req_id}} = State) ->
-    do_setopts(Socket, [{active, once}], State),
+    ok = do_setopts(Socket, [{active, once}], State),
     {noreply, set_inac_timer(State)};
 
 handle_info({stream_next, _Req_id}, State) ->
@@ -298,7 +298,7 @@ handle_sock_data(Data, #state{status = get_header}=State) ->
         #state{socket = Socket, status = Status, cur_req = CurReq} = State_1 ->
             case {Status, CurReq} of
                 {get_header, #request{caller_controls_socket = true}} ->
-                    do_setopts(Socket, [{active, once}], State_1);
+                    ok = do_setopts(Socket, [{active, once}], State_1);
                 _ ->
                     active_once(State_1)
             end,
@@ -337,7 +337,7 @@ handle_sock_data(Data, #state{status           = get_body,
                         true ->
                             active_once(State_1);
                         false when Ccs == true ->
-                            do_setopts(Socket, [{active, once}], State);
+                            ok = do_setopts(Socket, [{active, once}], State);
                         false ->
                             active_once(State_1)
                     end,
@@ -568,11 +568,6 @@ do_send(Req, #state{socket = Sock,
 do_send(Req, #state{socket = Sock, is_ssl = true})  ->  ssl:send(Sock, Req);
 do_send(Req, #state{socket = Sock, is_ssl = false}) ->  gen_tcp:send(Sock, Req).
 
-%% @spec do_send_body(Sock::socket_descriptor(), Source::source_descriptor(), IsSSL::boolean()) -> ok | error()
-%% source_descriptor() = fun_arity_0           |
-%%                       {fun_arity_0}         |
-%%                       {fun_arity_1, term()}
-%% error() = term()
 do_send_body(Source, State, TE) when is_function(Source) ->
     do_send_body({Source}, State, TE);
 do_send_body({Source}, State, TE) when is_function(Source) ->
@@ -589,30 +584,47 @@ do_send_body(Body, State, _TE) ->
             Ret
     end.
 
-generate_body({Source, Source_state} = In) ->
+generate_body({Source, Source_state} = In) when is_function(Source) ->
     case Source(Source_state) of
         {ok, Data, Source_state_1} ->
             {{ok, Data}, {Source, Source_state_1}};
         Ret ->
             {Ret, In}
     end;
-generate_body(Source) ->
+generate_body(Source) when is_function(Source) ->
     {Source(), Source}.
 
-do_send_body_1({Resp, Source}, State, TE, Acc) ->
+do_send_body_1({Resp, Source}, State, TE, Acc) when is_function(Source) ->
     case Resp of
         {ok, Data} when Data == []; Data == <<>> ->
-             do_send_body_1(generate_body(Source), State, TE, Acc);
+            do_send_body_1(generate_body(Source), State, TE, Acc);
         {ok, Data} ->
-            Data_1 = maybe_chunked_encode(Data, TE),
-            do_send(Data_1, State),
-            do_send_body_1(generate_body(Source), State, TE, [Data_1 | Acc]);
+            Acc_1 = case TE of
+                        true ->
+                            ok = do_send(maybe_chunked_encode(Data, TE), State),
+                            Acc;
+                        false ->
+                            [Data | Acc]
+                    end,
+            do_send_body_1(generate_body(Source), State, TE, Acc_1);
+        {ok, Data, New_source_state} when Data == []; Data == <<>> ->
+            do_send_body_1(generate_body({Source, New_source_state}), State, TE, Acc);
+        {ok, Data, New_source_state} ->
+            Acc_1 = case TE of
+                        true ->
+                            ok = do_send(maybe_chunked_encode(Data, TE), State),
+                            Acc;
+                        false ->
+                            [Data | Acc]
+                    end,
+            do_send_body_1(generate_body({Source, New_source_state}), State, TE, Acc_1);
         eof when TE == true ->
-            Body = <<"0\r\n\r\n">>,
-            do_send(Body, State),
-            {ok, list_to_binary(lists:reverse([Body | Acc]))};
+            ok = do_send(<<"0\r\n\r\n">>, State),
+            {ok, []};
         eof ->
-            {ok, list_to_binary(lists:reverse(Acc))};
+            Body = list_to_binary(lists:reverse(Acc)),
+            ok = do_send(Body, State),
+            {ok, Body};
         Err ->
             Err
     end.
@@ -634,7 +646,7 @@ do_close(#state{socket = Sock, is_ssl = false}) ->  catch gen_tcp:close(Sock).
 active_once(#state{cur_req = #request{caller_controls_socket = true}}) ->
     ok;
 active_once(#state{socket = Socket} = State) ->
-    do_setopts(Socket, [{active, once}], State).
+    ok = do_setopts(Socket, [{active, once}], State).
 
 do_setopts(_Sock, [],   _)    ->  ok;
 do_setopts(Sock, Opts, #state{is_ssl = true,
@@ -725,7 +737,7 @@ send_req_1(From,
             case do_send_body(Body_1, State_1, TE) of
                 {ok, _Sent_body} ->
                     trace_request_body(Body_1),
-                    active_once(State_1),
+                    ok = active_once(State_1),
                     State_1_1 = inc_pipeline_counter(State_1),
                     State_2 = State_1_1#state{status     = get_header,
                                               cur_req    = NewReq,
@@ -807,7 +819,7 @@ send_req_1(From,
                       timer_ref              = Ref
                      },
     trace_request(Req),
-    do_setopts(Socket, Caller_socket_options, State),
+    ok = do_setopts(Socket, Caller_socket_options, State),
     TE = is_chunked_encoding_specified(Options),
     case do_send(Req, State) of
         ok ->
@@ -1165,6 +1177,25 @@ parse_response(Data, #state{reply_buffer = Acc, reqs = Reqs,
                                ConnClose =:= "close" ->
                     send_async_headers(ReqId, StreamTo, Give_raw_headers, State_1),
                     State_1#state{reply_buffer = Data_1};
+                undefined when StatCode =:= "303" ->
+                    %% Some servers send 303 requests without a body.
+                    %% RFC2616 says that they SHOULD, but they dont.
+                    case ibrowse:get_config_value(allow_303_with_no_body, false) of
+                      false -> 
+                        fail_pipelined_requests(State_1,
+                                            {error, {content_length_undefined,
+                                                     {stat_code, StatCode}, Headers}}),
+                       {error, content_length_undefined};
+                      true -> 
+                        {_, Reqs_1} = queue:out(Reqs),
+                        send_async_headers(ReqId, StreamTo, Give_raw_headers, State_1),
+                        State_1_1 = do_reply(State_1, From, StreamTo, ReqId, Resp_format,
+                                             {ok, StatCode, Headers_1, []}),
+                        cancel_timer(T_ref, {eat_message, {req_timedout, From}}),
+                        State_2 = reset_state(State_1_1),
+                        State_3 = set_cur_request(State_2#state{reqs = Reqs_1}),
+                        parse_response(Data_1, State_3)
+                    end;
                 undefined ->
                     fail_pipelined_requests(State_1,
                                             {error, {content_length_undefined,
@@ -1457,7 +1488,7 @@ set_cur_request(#state{reqs = Reqs, socket = Socket} = State) ->
         {value, #request{caller_controls_socket = Ccs} = NextReq} ->
             case Ccs of
                 true ->
-                    do_setopts(Socket, [{active, once}], State);
+                    ok = do_setopts(Socket, [{active, once}], State);
                 _ ->
                     ok
             end,
@@ -1914,7 +1945,8 @@ dec_pipeline_counter(#state{cur_pipeline_size = Pipe_sz,
                             lb_ets_tid = Tid} = State) ->
     try
         update_counter(Tid, self(), {2,-1,0,0}),
-        update_counter(Tid, self(), {3,-1,0,0})
+        update_counter(Tid, self(), {3,-1,0,0}),
+        ok
     catch
         _:_ ->
             ok
@@ -1990,5 +2022,7 @@ trace_request_body(Body) ->
             ok
     end.
 
-to_binary(X) when is_list(X)   -> list_to_binary(X);
-to_binary(X) when is_binary(X) -> X.
+to_binary({X, _}) when is_function(X) -> to_binary(X);
+to_binary(X) when is_function(X)      -> <<"body generated by function">>;
+to_binary(X) when is_list(X)          -> list_to_binary(X);
+to_binary(X) when is_binary(X)        -> X.

src/ibrowse_socks5.erl

@@ -22,6 +22,7 @@ connect(Host, Port, Options) ->
 
     {ok, Socket} = gen_tcp:connect(Socks5Host, Socks5Port, [binary, {packet, 0}, {keepalive, true}, {active, false}]),
 
+    {ok, _Bin} =
     case proplists:get_value(socks5_user, Options, undefined) of
         undefined ->
             ok = gen_tcp:send(Socket, <<?SOCKS5, 1, ?AUTH_METHOD_NO>>),

test/ibrowse_test.erl

@@ -27,7 +27,12 @@
          test_head_transfer_encoding/0,
          test_head_transfer_encoding/1,
          test_head_response_with_body/0,
-         test_head_response_with_body/1
+         test_head_response_with_body/1,
+         test_303_response_with_no_body/0,
+         test_303_response_with_no_body/1,
+         test_303_response_with_a_body/0,
+         test_303_response_with_a_body/1,
+         test_generate_body_0/0
 	]).
 
 test_stream_once(Url, Method, Options) ->
@@ -233,7 +238,9 @@ dump_errors(Key, Iod) ->
                     {local_test_fun, test_20122010, []},
                     {local_test_fun, test_pipeline_head_timeout, []},
                     {local_test_fun, test_head_transfer_encoding, []},
-                    {local_test_fun, test_head_response_with_body, []}
+                    {local_test_fun, test_head_response_with_body, []},
+                    {local_test_fun, test_303_response_with_a_body, []}
+
 		   ]).
 
 unit_tests() ->
@@ -476,6 +483,37 @@ test_head_response_with_body(Url) ->
             {test_failed, Res}
     end.
 
+%%------------------------------------------------------------------------------
+%% Test what happens when a 303 response has no body
+%% Github issue #97 
+%% ------------------------------------------------------------------------------
+test_303_response_with_no_body() ->
+    clear_msg_q(),
+    test_303_response_with_no_body("http://localhost:8181/ibrowse_303_no_body_test").
+
+test_303_response_with_no_body(Url) ->
+    ibrowse:add_config([{allow_303_with_no_body, true}]),
+    case ibrowse:send_req(Url, [], post) of
+        {ok, "303", _, _} ->
+            success;
+        Res ->
+            {test_failed, Res}
+    end.
+
+%% Make sure we don't break requests that do have a body.
+test_303_response_with_a_body() ->
+    clear_msg_q(),
+    test_303_response_with_no_body("http://localhost:8181/ibrowse_303_with_body_test").
+
+test_303_response_with_a_body(Url) ->
+    ibrowse:add_config([{allow_303_with_no_body, true}]),
+    case ibrowse:send_req(Url, [], post) of
+        {ok, "303", _, "abcde"} ->
+            success;
+        Res ->
+            {test_failed, Res}
+    end.
+
 %%------------------------------------------------------------------------------
 %% Test what happens when the request at the head of a pipeline times out
 %%------------------------------------------------------------------------------
@@ -616,6 +654,46 @@ do_test_20122010_1(Expected_resp, Req_id, Acc) ->
             exit({timeout, test_failed})
     end.
 
+%%------------------------------------------------------------------------------
+%% Test requests where body is generated using a Fun
+%%------------------------------------------------------------------------------
+test_generate_body_0() ->
+    io:format("Testing that generation of body using fun works...~n", []),
+    Tid = ets:new(ibrowse_test_state, [public]),
+    try
+        Body_1 = <<"Part 1 of the body">>,
+        Body_2 = <<"Part 2 of the body\r\n\r\n">>,
+        Size = size(Body_1) + size(Body_2),
+        Body = list_to_binary([Body_1, Body_2]),
+        Fun = fun() ->
+                      case ets:lookup(Tid, body_gen_state) of
+                          [] ->
+                              ets:insert(Tid, {body_gen_state, 1}),
+                              {ok, Body_1};
+                          [{_, 1}]->
+                              ets:insert(Tid, {body_gen_state, 2}),
+                              {ok, Body_2};
+                          [{_, 2}] ->
+                              eof
+                      end
+              end,
+        case ibrowse:send_req("http://localhost:8181/echo_body",
+                              [{"Content-Length", Size}],
+                              post,
+                              Fun,
+                              [{response_format, binary},
+                               {http_vsn, {1,0}}]) of
+            {ok, "200", _, Body} ->
+                io:format("    Success~n", []),
+                success;
+            Err ->
+                io:format("Test failed : ~p~n", [Err]),
+                {test_failed, Err}
+        end
+    after
+        ets:delete(Tid)
+    end.
+
 do_trace(Fmt, Args) ->
     do_trace(get(my_trace_flag), Fmt, Args).
 

test/ibrowse_test_server.erl

@@ -161,6 +161,18 @@ process_request(Sock, Sock_type,
                          uri = {abs_path, "/ibrowse_head_test"}}) ->
     Resp = <<"HTTP/1.1 200 OK\r\nServer: Apache-Coyote/1.1\r\nTransfer-Encoding: chunked\r\nDate: Wed, 04 Apr 2012 16:53:49 GMT\r\nConnection: close\r\n\r\n">>,
     do_send(Sock, Sock_type, Resp);
+process_request(Sock, Sock_type,
+                #request{method='POST',
+                         headers = _Headers,
+                         uri = {abs_path, "/ibrowse_303_no_body_test"}}) ->
+    Resp = <<"HTTP/1.1 303 See Other\r\nLocation: http://example.org\r\n">>,
+    do_send(Sock, Sock_type, Resp);
+process_request(Sock, Sock_type,
+                #request{method='POST',
+                         headers = _Headers,
+                         uri = {abs_path, "/ibrowse_303_with_body_test"}}) ->
+    Resp = <<"HTTP/1.1 303 See Other\r\nLocation: http://example.org\r\nContent-Length: 5\r\n\r\nabcde">>,
+    do_send(Sock, Sock_type, Resp);
 process_request(Sock, Sock_type, Req) ->
     do_trace("Recvd req: ~p~n", [Req]),
     Resp = <<"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n">>,