Added ability to generate Chunked requests (see option transfer_encoding in doc). If destination port is 80, it is not included in the Host header

README

@@ -1,10 +1,11 @@
-$Id: README,v 1.3 2005/05/08 22:04:19 chandrusf Exp $
+$Id: README,v 1.4 2005/12/08 12:05:06 chandrusf Exp $
 
 ibrowse is a HTTP client. The following are a list of features.
 	- RFC2616 compliant (AFAIK) 
 	- supports GET, POST, OPTIONS, HEAD, PUT, DELETE, TRACE only
 	- Understands HTTP/0.9, HTTP/1.0 and HTTP/1.1
 	- Understands chunked encoding
+        - Can generate requests using Chunked Transfer-Encoding
 	- Named pools of connections to each webserver
 	- Pipelining support
 	- Download to file
@@ -16,11 +17,18 @@ ibrowse is a HTTP client. The following are a list of features.
 
 Comments to : Chandrashekhar.Mullaparthi@t-mobile.co.uk
 
-CONTRIBUTIONS
-=============
+CONTRIBUTIONS & CHANGE HISTORY
+==============================
 08-May-2005 - Youns Hafri made a CRUX LINUX port of ibrowse.
 	      http://yhafri.club.fr/crux/index.html
 
+22-Nov-2005 - Added ability to generate requests using the Chunked 
+              Transfer-Encoding.
+
+08-Dec-2005 - Richard Cameron (camster@citeulike.org). Patch to ibrowse to
+              prevent port number being included in the Host header when port
+              80 is intended.
+
 Here are some usage examples. Enjoy!
 
 5> ibrowse:start().

src/ibrowse.erl

@@ -58,7 +58,7 @@
 %% driver isn't actually used.</p>
 
 -module(ibrowse).
--vsn('$Id: ibrowse.erl,v 1.1 2005/05/05 22:28:28 chandrusf Exp $ ').
+-vsn('$Id: ibrowse.erl,v 1.2 2005/12/08 12:05:07 chandrusf Exp $ ').
 
 -behaviour(gen_server).
 %%--------------------------------------------------------------------
@@ -181,7 +181,7 @@ send_req(Url, Headers, Method, Body) ->
 %% HTTP Version to use is not specified, the default is 1.1
 %% @spec send_req(Url::string(), Headers::headerList(), Method::method(), Body::body(), Options::optionList()) -> response()
 %% optionList() = [option()]
-%% option() = {max_sessions, integer()}        | 
+%% option() = {max_sessions, integer()}        |
 %%          {max_pipeline_size, integer()}     |
 %%          {trace, boolean()}                 | 
 %%          {is_ssl, boolean()}                |
@@ -198,11 +198,14 @@ send_req(Url, Headers, Method, Body) ->
 %%          {content_type, string()}           |
 %%          {save_response_to_file, boolean()} |
 %%          {stream_to, process()}             |
-%%          {http_vsn, {MajorVsn, MinorVsn}}
+%%          {http_vsn, {MajorVsn, MinorVsn}}   |
+%%          {transfer_encoding, {chunked, ChunkSize}}
+%% 
 %% process() = pid() | atom()
 %% username() = string()
 %% password() = string()
 %% SSLOpt = term()
+%% ChunkSize = integer()
 send_req(Url, Headers, Method, Body, Options) ->
     send_req(Url, Headers, Method, Body, Options, 30000).
 

src/ibrowse_http_client.erl

@@ -6,7 +6,7 @@
 %%% Created : 11 Oct 2003 by Chandrashekhar Mullaparthi <chandrashekhar.mullaparthi@t-mobile.co.uk>
 %%%-------------------------------------------------------------------
 -module(ibrowse_http_client).
--vsn('$Id: ibrowse_http_client.erl,v 1.3 2005/06/07 21:40:02 chandrusf Exp $ ').
+-vsn('$Id: ibrowse_http_client.erl,v 1.4 2005/12/08 12:05:07 chandrusf Exp $ ').
 
 -behaviour(gen_server).
 %%--------------------------------------------------------------------
@@ -451,8 +451,12 @@ send_req_1(Url, Headers, Method, Body, Options, Sock, State) ->
 	 port = Port, 
 	 path = RelPath} = Url_1 = parse_url(Url),
     Headers_1 = add_auth_headers(Url_1, Options, Headers, State),
+    HostString = case Port of
+		     80 -> Host;
+		     _ -> [Host, ":", integer_to_list(Port)]
+		 end,
     Req = make_request(Method, 
-		       [{"Host", [Host, ":", integer_to_list(Port)]} | Headers_1], 
+		       [{"Host", HostString} | Headers_1], 
 		       AbsPath, RelPath, Body, Options, State#state.use_proxy),
     case get(my_trace_flag) of %%Avoid the binary operations if trace is not on...
 	true ->
@@ -529,14 +533,26 @@ make_request(Method, Headers, AbsPath, RelPath, Body, Options, UseProxy) ->
 		    true ->
 			Headers
 		end,
-    Headers_2 = cons_headers(Headers_1),
+    {Headers_2, Body_1} = 
+	case get_value(transfer_encoding, Options, false) of
+	    false ->
+		{Headers_1, Body};
+	    {chunked, ChunkSize} ->
+		{[{X, Y} || {X, Y} <- Headers_1, 
+			    X /= "Content-Length",
+			    X /= "content-length",
+			    X /= content_length] ++
+		 [{"Transfer-Encoding", "chunked"}],
+		 chunk_request_body(Body, ChunkSize)}
+	end,
+    Headers_3 = cons_headers(Headers_2),
     Uri = case get_value(use_absolute_uri, Options, false) or UseProxy of
 	      true ->
 		  AbsPath;
 	      false -> 
 		  RelPath
 	  end,
-    [method(Method), " ", Uri, " ", HttpVsn, crnl(), Headers_2, crnl(), Body].
+    [method(Method), " ", Uri, " ", HttpVsn, crnl(), Headers_3, crnl(), Body_1].
 
 http_vsn_string({0,9}) -> "HTTP/0.9";
 http_vsn_string({1,0}) -> "HTTP/1.0";
@@ -568,6 +584,38 @@ encode_headers([{Name,Val} | T], Acc) when atom(Name) ->
 encode_headers([], Acc) ->
     lists:reverse(Acc).
 
+chunk_request_body(Body, ChunkSize) ->
+    chunk_request_body(Body, ChunkSize, []).
+
+chunk_request_body(Body, _ChunkSize, Acc) when Body == <<>>; Body == [] ->
+    LastChunk = "0\r\n",
+    lists:reverse(["\r\n", LastChunk | Acc]);
+chunk_request_body(Body, ChunkSize, Acc) when binary(Body),
+                                              size(Body) >= ChunkSize ->
+    <<ChunkBody:ChunkSize/binary, Rest/binary>> = Body,
+    Chunk = [ibrowse_lib:dec2hex(4, ChunkSize),"\r\n",
+	     ChunkBody, "\r\n"],
+    chunk_request_body(Rest, ChunkSize, [Chunk | Acc]);
+chunk_request_body(Body, _ChunkSize, Acc) when binary(Body) ->
+    BodySize = size(Body),
+    Chunk = [ibrowse_lib:dec2hex(4, BodySize),"\r\n",
+	     Body, "\r\n"],
+    LastChunk = "0\r\n",
+    lists:reverse(["\r\n", LastChunk, Chunk | Acc]);
+chunk_request_body(Body, ChunkSize, Acc) when list(Body),
+                                              length(Body) >= ChunkSize ->
+    {ChunkBody, Rest} = split_list_at(Body, ChunkSize),
+    Chunk = [ibrowse_lib:dec2hex(4, ChunkSize),"\r\n",
+	     ChunkBody, "\r\n"],
+    chunk_request_body(Rest, ChunkSize, [Chunk | Acc]);
+chunk_request_body(Body, _ChunkSize, Acc) when list(Body) ->
+    BodySize = length(Body),
+    Chunk = [ibrowse_lib:dec2hex(4, BodySize),"\r\n",
+	     Body, "\r\n"],
+    LastChunk = "0\r\n",
+    lists:reverse(["\r\n", LastChunk, Chunk | Acc]).
+
+
 parse_response(_Data, #state{cur_req = undefined}=State) ->
     State#state{status = idle};
 parse_response(Data, #state{reply_buffer=Acc, reqs=Reqs,

src/ibrowse_lib.erl

@@ -5,7 +5,7 @@
 %% @doc Module with a few useful functions
 
 -module(ibrowse_lib).
--vsn('$Id: ibrowse_lib.erl,v 1.2 2005/05/08 22:04:19 chandrusf Exp $ ').
+-vsn('$Id: ibrowse_lib.erl,v 1.3 2005/12/08 12:05:07 chandrusf Exp $ ').
 -author('chandru').
 -ifdef(debug).
 -compile(export_all).
@@ -14,6 +14,7 @@
 -export([url_encode/1,
 	 decode_rfc822_date/1,
 	 status_code/1,
+	 dec2hex/2,
 	 drv_ue/1,
 	 drv_ue/2]).
 
@@ -138,3 +139,13 @@ status_code(504) -> gateway_timeout;
 status_code(505) -> http_version_not_supported;
 status_code(X) when is_list(X) -> status_code(list_to_integer(X));
 status_code(_)   -> unknown_status_code.
+
+%% @doc dec2hex taken from gtk.erl in std dist
+%% @spec dec2hex(M, N) -> string()
+%% M = integer() - number of hex digits required
+%% N = integer() - the number to represent as hex
+dec2hex(M,N) -> dec2hex(M,N,[]).
+
+dec2hex(0,N,Ack) -> Ack;
+dec2hex(M,N,Ack) -> dec2hex(M-1,N bsr 4,[d2h(N band 15)|Ack]).
+