[erlang-questions] Issues with SSL Sockets and HTTPC

[Eric Boyer]

Hi all,

I've got a webmachine/mochiweb server up and running accepting SSL
connections to transfer files via a streamed body.  There are 2 issues that
I'm seeing:

1. Very often the sockets close themselves 'randomly', approximately 5% of
the time. Resulting in errors during the write process.  Code is at the end
of the email.

{error,{error,{badmatch,{error,closed}},
              [{webmachine_request,recv_chunked_body,4},
               {file_srv,stream_data_to_disk,2},
               {file_srv,accept_content,2},
               {webmachine_resource,resource_call,3},
               {webmachine_resource,do,3},
               {webmachine_decision_core,resource_call,1},
               {webmachine_decision_core,accept_helper,0},
               {webmachine_decision_core,decision,1}]}}

{error,{exit,normal,
             [{webmachine_request,read_chunk_length,3},
              {webmachine_request,recv_chunked_body,3},
              {file_srv,stream_data_to_disk,2},
              {file_srv,accept_content,2},
              {webmachine_resource,resource_call,3},
              {webmachine_resource,do,3},
              {webmachine_decision_core,resource_call,1},
              {webmachine_decision_core,accept_helper,0}]}}

These errors both stem from read from the socket. Now my question is, why
are closed socket errors occurring so often? How would I go about trying to
mitigate and/or prevent these issues from happening?

2. I'm using the 'chunkify' option on the httpc request which allows a
stream of data to the webserver, the issue that I'm seeing is that when the
connection is forcibly shutdown (ssl socket read error, issue #1), httpc
will sometimes attempt another request of the same type automatically!
There doesn't seem to be anything that I can see that controls this
behaviour. Is there any way to turn this off? Is there something that I'm
doing wrong? It seems more like a bug than anything else as it only happens
occasionally.

127.0.0.1 - - [08/Nov/2011:18:14:55 -0500] "HEAD /files/test19.txt
HTTP/1.1" 200 chunked "" ""
127.0.0.1 - - [08/Nov/2011:18:14:55 -0500] "PUT /files/test19.txt HTTP/1.1"
500 697 "" ""
127.0.0.1 - - [08/Nov/2011:18:14:55 -0500] "PUT /files/test19.txt HTTP/1.1"
500 697 "" ""

This error report is generated on the duplicate retry, but not when it
functions properly:

=ERROR REPORT==== 8-Nov-2011::18:29:47 ===
Received unexpected ssl data on {sslsocket,new_ssl,<0.850.0>}
   Data:       <<"HTTP/1.1 500 Internal Server Error\r\nServer:
MochiWeb/1.1 WebMachine/1.9.0 (someone had painted it blue)\r\nDate: Tue,
08 Nov 2011 23:29:47 GMT\r\nContent-Type: text/html\r\nContent-Length:
697\r\n\r\n">>
   MFA:        undefined
   Request:    undefined
   Session:    {session,{{"localhost",443},<0.849.0>},
                        false,https,
                        {sslsocket,new_ssl,<0.850.0>},
                        {essl,[]},
                        1,keep_alive}
   Status:     keep_alive
   StatusLine: undefined
   Profile:    httpc_manager


Thanks for reading! Any tips would be appreciated.

Cheers,
Eric Boyer



==CODE==
Server (Webmachine):

content_types_accepted(ReqData, Context) ->
   CT = case wrq:get_req_header("content-type", ReqData) of
             undefined -> "application/octet-stream";
             X -> X
         end,
    {MT, _Params} = webmachine_util:media_type_to_detail(CT),
    {[{MT, accept_content}], ReqData, Context}.

accept_content(ReqData, Context) ->
   Path = "/some/path/",
  stream_data_to_disk(wrq:stream_req_body(ReqData, ?CHUNK), Path, Start);

stream_data_to_disk(Data, File, Position) ->
  {ok, FH} = file:open(File, [binary, raw, write, read]),
  stream_data_to_disk(Data, FH).

stream_data_to_disk({Hunk, done}, File) ->
file:write(File, Hunk),
file:close(File);

stream_data_to_disk({Hunk, Next}, File) ->
file:write(File, Hunk),
stream_data_to_disk(Next(), File).

Client:

read_file({file, Name, Pid, RemotePos}) ->
case file:open(Name, [read, raw, binary]) of
{ok, FD} ->
Context = utils:get_md5_context(FD, RemotePos),
read_file({fd, FD, Pid, Context});
{error, _} ->
eof
end;
read_file({fd, FD, Pid, Cntx}) ->
case file:read(FD, ?CHUNK) of
{ok, Data} ->
NewContext = crypto:md5_update(Cntx, Data),
{ok, Data, {fd, FD, Pid, NewContext}};
eof ->
file:close(FD),
Pid ! {hash, crypto:md5_final(Cntx)},
eof;
_ ->
eof
end.

transmit(FileName, Url, _, Size) ->
Pid = self(),
RemotePos = get_size(filename:basename(FileName), Url),
httpc:request(put,
{url(Url, filename:basename(FileName)),
 [range_header(RemotePos, Size - RemotePos, Size)],
 "application/octet-stream",
{chunkify, fun read_file/1, {file, FileName, Pid, RemotePos}}},[{ssl,
[{reuse_sessions, true}]}], [{body_format, binary}] )
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://erlang.org/pipermail/erlang-questions/attachments/20111108/42138c7f/attachment.htm>