issue with http:request

Alexander Zhuravlev a.zhuravlev@REDACTED
Wed Feb 10 14:08:51 CET 2010


On 2/10/10 12:54 PM, Alexander Zhuravlev wrote:
> Hello,
>
> It looks like there is some issue with http request handling in
> http:request function or some other subsystem underneath.
>
> Here is a test case:
>
> Erlang R13B03 (erts-5.7.4) [source] [smp:2:2] [rq:2] [async-threads:0] [hipe] [kernel-poll:false]
>
> 1>  inets:start().
> ok
> 2>  % this one works as expected
> 2>  http:request(head, {"http://google.com", []}, [{timeout, 100}], []).
> {error,timeout}
> 3>  URL = [104,116,116,112,58,47,47,227,129,147,227,130,140,230,166,130,227,
> 3>  129,173,229,189,147,227,129,159,227,129,163,227,129,166,227,130,
> 3>  139,227,128,130,119,119,119,46,112,97,103,101,46,115,97,110,110,
> 3>  101,116,46,110,101,46,106,112,47,109,97,121,117,114,105,47,122,121,
> 3>  111,115,101,105,47,109,97,114,95,51,48,54,46,104,116,109,108].
> [104,116,116,112,58,47,47,227,129,147,227,130,140,230,166,
>   130,227,129,173,229,189,147,227,129,159,227,129,163,227|...]
> 4>  http:request(head, {URL, []}, [{timeout, 100}], []).
>
> The latest request hangs indefinitely.
> Request to the same URL (while being UTF-8 encoded) also hangs:
>
> 6>  http:request(head, {"http://これ概ね当たってる。www.page.sannet.ne.jp/mayuri/zyosei/mar_306.html", []}, [{timeout, 100}], []).
>
> Moreover, http_uri shows that both URLs are valid:
>
> 1>  http_uri:parse(URL).
> {http,[], [227,129,147,227,130,140,230,166,130,227,129,173,229,189,
>               147,227,129,159,227,129,163,227,129,166,227,130|...],
> 	           80,"/mayuri/zyosei/mar_306.html",[]}
>
> 2>
> http_uri:parse("http://これ概ね当たってる。www.page.sannet.ne.jp/mayuri/zyosei/mar_306.html").
> {http,[],
>        [12371,12428,27010,12397,24403,12383,12387,12390,12427,
>               12290,119,119,119,46,112,97,103,101,46,115,97,110,110,101,
> 	            116,46|...], 80,"/mayuri/zyosei/mar_306.html",[]}
>
>
> Our production system logs the following error to the console (however I do not see it when I run the
> code in a new local erlang shell), I suppose it is connected to the issue above:
>
> =CRASH REPORT==== 10-Feb-2010::12:25:09 ===
>    crasher:
>      initial call: httpc_handler:init/1
>      pid:<0.18577.0>
>      registered_name: []
>      exception exit: badarg
>        in function  gen_tcp:connect/4
>        in call from httpc_handler:send_first_request/3
>        in call from httpc_handler:init/1
>      ancestors: [httpc_handler_sup,httpc_sup,inets_sup,<0.80.0>]
>      messages: []
>      links: [<0.86.0>]
>      dictionary: []
>      trap_exit: true
>      status: running
>      heap_size: 233
>      stack_size: 24
>      reductions: 104
>    neighbours:
>
> =SUPERVISOR REPORT==== 10-Feb-2010::12:25:09 ===
>       Supervisor: {local,httpc_handler_sup}
>       Context:    child_terminated
>       Reason:     badarg
>       Offender:   [{pid,<0.18577.0>},
>                    {name,undefined},
>                    {mfa,
>                        {httpc_handler,start_link,
>                            [{request,#Ref<0.0.0.79940>,<0.18533.0>,0,http,
>                                 {[12371,12428,27010,12397,24403,12383,12387,
>                                   12390,12427,12290,119,119,119,46,112,97,103,
>                                   101,46,115,97,110,110,101,116,46,110,101,46,
>                                   106,112],
>                                  80},
>                                 "/mayuri/zyosei/mar_306.html",[],get,
>                                 {http_request_h,undefined,"keep-alive",
>                                     undefined,undefined,undefined,undefined,
>                                     undefined,undefined,undefined,undefined,
>                                     undefined,undefined,undefined,undefined,
>                                     undefined,undefined,
>                                     [12371,12428,27010,12397,24403,12383,12387,
>                                      12390,12427,12290,119,119,119,46,112,97,
>                                      103,101,46,115,97,110,110,101,116,46,110,
>                                      101,46,106,112],
>                                     undefined,undefined,undefined,undefined,
>                                     undefined,undefined,undefined,undefined,
>                                     undefined,[],undefined,undefined,undefined,
>                                     undefined,"0",undefined,undefined,
>                                     undefined,undefined,undefined,undefined,[]},
>                                 {[],[]},
>                                 {http_options,"HTTP/1.1",infinity,true,[],
>                                     undefined,false,infinity},
>                                 [104,116,116,112,58,47,47,12371,12428,27010,
>                                  12397,24403,12383,12387,12390,12427,12290,119,
>                                  119,119,46,112,97,103,101,46,115,97,110,110,
>                                  101,116,46,110,101,46,106,112,47,109,97,121,
>                                  117,114,105,47,122,121,111,115,101,105,47,109,
>                                  97,114,95,51,48,54,46,104,116,109,108],
>                                 [],none,[]},
>                             {options,
>                                 {undefined,[]},
>                                 0,2,5,120000,2,disabled,false,inet,default,
>                                 default},
>                             httpc_manager]}},
>                    {restart_type,temporary},
>                    {shutdown,4000},
>                    {child_type,worker}]
>

We have managed to deploy a workaround for the issue with request 
hanging using a patch from:

http://www.erlang.org/cgi-bin/ezmlm-cgi/2/1601

Now the http:request(head, {URL, []}, [{timeout, 5000}], []) function 
call returns:

{error, internal_error}

Here is the patch in the unified format:

--- httpc_manager.erl	2009-11-10 19:06:40.000000000 +0300
+++ /home/zaa/httpc_manager.erl	2010-02-10 15:22:23.000000000 +0300
@@ -363,6 +363,11 @@
      %% Handled in DOWN
      {noreply, State};
  handle_info({'DOWN', _, _, Pid, _}, State) ->
+    Requests = ets:match(State#state.handler_db, {'$1', Pid, '$2'}),
+    [begin
+	httpc_response:send(From, {Id, {error, internal_error}}) end ||
+		[Id, From] <- Requests],
+
      ets:match_delete(State#state.handler_db, {'_', Pid, '_'}),

      %% If there where any canceled request, handled by the
@@ -500,10 +505,31 @@
  	    ets:insert(State#state.handler_db, {Request#request.id,
  						HandlerPid,
  						Request#request.from});
-	_  -> %timeout pipelining failed
+	Error  -> %timeout pipelining failed
+	    case Error of
+		{request_failed, _Reason, Queue} ->
+			spawn(fun() -> restart_requests(Queue, State) end);
+		_E ->
+			ok
+	    end,
  	    start_handler(Request, State)
      end.

+restart_requests([], _) -> ok;
+restart_requests([undefined | Queue], State) ->
+    restart_requests(Queue, State);
+restart_requests([Request = #request{ from = answer_sent} | Queue], 
State) ->
+    restart_requests(Queue, State);
+restart_requests([Request | Queue], State) ->
+    restart_request(Request, State),
+    restart_requests(Queue, State).
+
+restart_request(Request, State) ->
+    ProfileName = State#state.profile_name,
+    catch ets:delete(State#state.handler_db, Request#request.id),
+    httpc_manager:request(Request, ProfileName).
+
+
  start_handler(Request, State) ->
      {ok, Pid} =
  	case is_inets_manager() of
--- httpc_handler.erl	2009-11-10 19:06:40.000000000 +0300
+++ /home/zaa/httpc_handler.erl	2010-02-10 15:22:23.000000000 +0300
@@ -284,7 +284,7 @@
  						  undefined}}}
  	    end;
  	{error, Reason} ->
-	    {reply, {pipline_failed, Reason}, State}
+	    return_error(pipline_failed, Reason, State)
      end;

  handle_call(Request, _, #state{session = Session =
@@ -337,9 +337,17 @@
  					    Relaxed]}}}
  	    end;
  	{error, Reason}    ->
-	    {reply, {request_failed, Reason}, State}
+	    return_error(request_failed, Reason, State)
      end.

+return_error(Error, Reason, State) ->
+    Queue = case State#state.status of
+	pipeline -> State#state.pipeline;
+	keep_alive -> State#state.keep_alive;
+	_ -> []
+    end,
+    {reply, {Error, Reason, [State#state.request | 
queue:to_list(Queue)]}, State#state{ pipeline = queue:new(), keep_alive 
= queue:new() }}.
+
  %%--------------------------------------------------------------------
  %% Function: handle_cast(Msg, State) -> {noreply, State} |
  %%          {noreply, State, Timeout} |


More information about the erlang-questions mailing list