issue with http:request
Alexander Zhuravlev
a.zhuravlev@REDACTED
Wed Feb 10 14:08:51 CET 2010
On 2/10/10 12:54 PM, Alexander Zhuravlev wrote:
> Hello,
>
> It looks like there is some issue with http request handling in
> http:request function or some other subsystem underneath.
>
> Here is a test case:
>
> Erlang R13B03 (erts-5.7.4) [source] [smp:2:2] [rq:2] [async-threads:0] [hipe] [kernel-poll:false]
>
> 1> inets:start().
> ok
> 2> % this one works as expected
> 2> http:request(head, {"http://google.com", []}, [{timeout, 100}], []).
> {error,timeout}
> 3> URL = [104,116,116,112,58,47,47,227,129,147,227,130,140,230,166,130,227,
> 3> 129,173,229,189,147,227,129,159,227,129,163,227,129,166,227,130,
> 3> 139,227,128,130,119,119,119,46,112,97,103,101,46,115,97,110,110,
> 3> 101,116,46,110,101,46,106,112,47,109,97,121,117,114,105,47,122,121,
> 3> 111,115,101,105,47,109,97,114,95,51,48,54,46,104,116,109,108].
> [104,116,116,112,58,47,47,227,129,147,227,130,140,230,166,
> 130,227,129,173,229,189,147,227,129,159,227,129,163,227|...]
> 4> http:request(head, {URL, []}, [{timeout, 100}], []).
>
> The latest request hangs indefinitely.
> Request to the same URL (while being UTF-8 encoded) also hangs:
>
> 6> http:request(head, {"http://これ概ね当たってる。www.page.sannet.ne.jp/mayuri/zyosei/mar_306.html", []}, [{timeout, 100}], []).
>
> Moreover, http_uri shows that both URLs are valid:
>
> 1> http_uri:parse(URL).
> {http,[], [227,129,147,227,130,140,230,166,130,227,129,173,229,189,
> 147,227,129,159,227,129,163,227,129,166,227,130|...],
> 80,"/mayuri/zyosei/mar_306.html",[]}
>
> 2>
> http_uri:parse("http://これ概ね当たってる。www.page.sannet.ne.jp/mayuri/zyosei/mar_306.html").
> {http,[],
> [12371,12428,27010,12397,24403,12383,12387,12390,12427,
> 12290,119,119,119,46,112,97,103,101,46,115,97,110,110,101,
> 116,46|...], 80,"/mayuri/zyosei/mar_306.html",[]}
>
>
> Our production system logs the following error to the console (however I do not see it when I run the
> code in a new local erlang shell), I suppose it is connected to the issue above:
>
> =CRASH REPORT==== 10-Feb-2010::12:25:09 ===
> crasher:
> initial call: httpc_handler:init/1
> pid:<0.18577.0>
> registered_name: []
> exception exit: badarg
> in function gen_tcp:connect/4
> in call from httpc_handler:send_first_request/3
> in call from httpc_handler:init/1
> ancestors: [httpc_handler_sup,httpc_sup,inets_sup,<0.80.0>]
> messages: []
> links: [<0.86.0>]
> dictionary: []
> trap_exit: true
> status: running
> heap_size: 233
> stack_size: 24
> reductions: 104
> neighbours:
>
> =SUPERVISOR REPORT==== 10-Feb-2010::12:25:09 ===
> Supervisor: {local,httpc_handler_sup}
> Context: child_terminated
> Reason: badarg
> Offender: [{pid,<0.18577.0>},
> {name,undefined},
> {mfa,
> {httpc_handler,start_link,
> [{request,#Ref<0.0.0.79940>,<0.18533.0>,0,http,
> {[12371,12428,27010,12397,24403,12383,12387,
> 12390,12427,12290,119,119,119,46,112,97,103,
> 101,46,115,97,110,110,101,116,46,110,101,46,
> 106,112],
> 80},
> "/mayuri/zyosei/mar_306.html",[],get,
> {http_request_h,undefined,"keep-alive",
> undefined,undefined,undefined,undefined,
> undefined,undefined,undefined,undefined,
> undefined,undefined,undefined,undefined,
> undefined,undefined,
> [12371,12428,27010,12397,24403,12383,12387,
> 12390,12427,12290,119,119,119,46,112,97,
> 103,101,46,115,97,110,110,101,116,46,110,
> 101,46,106,112],
> undefined,undefined,undefined,undefined,
> undefined,undefined,undefined,undefined,
> undefined,[],undefined,undefined,undefined,
> undefined,"0",undefined,undefined,
> undefined,undefined,undefined,undefined,[]},
> {[],[]},
> {http_options,"HTTP/1.1",infinity,true,[],
> undefined,false,infinity},
> [104,116,116,112,58,47,47,12371,12428,27010,
> 12397,24403,12383,12387,12390,12427,12290,119,
> 119,119,46,112,97,103,101,46,115,97,110,110,
> 101,116,46,110,101,46,106,112,47,109,97,121,
> 117,114,105,47,122,121,111,115,101,105,47,109,
> 97,114,95,51,48,54,46,104,116,109,108],
> [],none,[]},
> {options,
> {undefined,[]},
> 0,2,5,120000,2,disabled,false,inet,default,
> default},
> httpc_manager]}},
> {restart_type,temporary},
> {shutdown,4000},
> {child_type,worker}]
>
We have managed to deploy a workaround for the issue with request
hanging using a patch from:
http://www.erlang.org/cgi-bin/ezmlm-cgi/2/1601
Now the http:request(head, {URL, []}, [{timeout, 5000}], []) function
call returns:
{error, internal_error}
Here is the patch in the unified format:
--- httpc_manager.erl 2009-11-10 19:06:40.000000000 +0300
+++ /home/zaa/httpc_manager.erl 2010-02-10 15:22:23.000000000 +0300
@@ -363,6 +363,11 @@
%% Handled in DOWN
{noreply, State};
handle_info({'DOWN', _, _, Pid, _}, State) ->
+ Requests = ets:match(State#state.handler_db, {'$1', Pid, '$2'}),
+ [begin
+ httpc_response:send(From, {Id, {error, internal_error}}) end ||
+ [Id, From] <- Requests],
+
ets:match_delete(State#state.handler_db, {'_', Pid, '_'}),
%% If there where any canceled request, handled by the
@@ -500,10 +505,31 @@
ets:insert(State#state.handler_db, {Request#request.id,
HandlerPid,
Request#request.from});
- _ -> %timeout pipelining failed
+ Error -> %timeout pipelining failed
+ case Error of
+ {request_failed, _Reason, Queue} ->
+ spawn(fun() -> restart_requests(Queue, State) end);
+ _E ->
+ ok
+ end,
start_handler(Request, State)
end.
+restart_requests([], _) -> ok;
+restart_requests([undefined | Queue], State) ->
+ restart_requests(Queue, State);
+restart_requests([Request = #request{ from = answer_sent} | Queue],
State) ->
+ restart_requests(Queue, State);
+restart_requests([Request | Queue], State) ->
+ restart_request(Request, State),
+ restart_requests(Queue, State).
+
+restart_request(Request, State) ->
+ ProfileName = State#state.profile_name,
+ catch ets:delete(State#state.handler_db, Request#request.id),
+ httpc_manager:request(Request, ProfileName).
+
+
start_handler(Request, State) ->
{ok, Pid} =
case is_inets_manager() of
--- httpc_handler.erl 2009-11-10 19:06:40.000000000 +0300
+++ /home/zaa/httpc_handler.erl 2010-02-10 15:22:23.000000000 +0300
@@ -284,7 +284,7 @@
undefined}}}
end;
{error, Reason} ->
- {reply, {pipline_failed, Reason}, State}
+ return_error(pipline_failed, Reason, State)
end;
handle_call(Request, _, #state{session = Session =
@@ -337,9 +337,17 @@
Relaxed]}}}
end;
{error, Reason} ->
- {reply, {request_failed, Reason}, State}
+ return_error(request_failed, Reason, State)
end.
+return_error(Error, Reason, State) ->
+ Queue = case State#state.status of
+ pipeline -> State#state.pipeline;
+ keep_alive -> State#state.keep_alive;
+ _ -> []
+ end,
+ {reply, {Error, Reason, [State#state.request |
queue:to_list(Queue)]}, State#state{ pipeline = queue:new(), keep_alive
= queue:new() }}.
+
%%--------------------------------------------------------------------
%% Function: handle_cast(Msg, State) -> {noreply, State} |
%% {noreply, State, Timeout} |
More information about the erlang-bugs
mailing list