[erlang-bugs] file:sendfile/5 bug and possible DOS attack

[Christopher Faulet]

Hi,

This summer, at my company, we encountered a problem that leads to the
VM hanging. After some painful investigation, we found that the problem
came from the file:sendfile/5 function.

When async threads are enabled, during a call to file:sendfile/5 the
efile driver sets the TCP socket in blocking mode. So an unresponsive
client can block the sendfile() syscall, thus blocking an async thread.
With few unresponsive clients, all async threads can be blocked and the
VM hangs (no more I/O are possible).

I attached 2 escripts to reproduce the bug:

* server - listen on a socket and wait a client connection to send a
large file using gen_tcp:send or file:sendfile:

  $> ./server 1234 /path/to/bigfile send
or
  $> ./server 1234 /path/to/bigfile sendfile

* slow_client - open a connection on the server and read incoming data
with a sleep of 10 seconds between each read:

  $> ./slow_client 127.0.0.1 1234

The server is started with 1 async thread. Every 2 seconds the server
tries to read the file info. When "sendfile" method is used, the call to
file:read_file_info/1 is blocked; all I/O are blocked, the VM is out.
When async threads are disabled or when the "send" method is used, there
is no problem. The file is sent (slowly) and the VM is still responsive.

So it is very easy to do a DOS attack on systems that use
file:sendfile/5 with async threads enabled. The problem comes from a
design choice of the efile driver. I have no solution to propose, but it
could be a good idea to add a warning in the documentation of
file:sendfile/5 (especially that the documentation encourages the use of
async threads).

Regards,
-- 
Christopher
-------------- next part --------------
#!/usr/bin/env escript
%% -*- erlang -*-
%%! -smp enable -sname server +A 1

-export([send/2, fstat/1, fstat_monitor/1]).

-include_lib("kernel/include/file.hrl").

main([Port0, File, Fun0]) ->
    try
        Port  = list_to_integer(Port0),
        Fun   = list_to_atom(Fun0),
        Opts  = [
                 binary,
                 {packet, 0},
                 {reuseaddr, true},
                 {active, false}
                ],
        spawn(node(), fun() -> fstat(File) end),
        case gen_tcp:listen(Port, Opts) of
            {ok, LSock} ->
                do_accept(LSock,
                          [{file, File}, {function, Fun}]);
            {error, Reason} ->
                io:format("Failed to listen on port ~p: ~p~n", [Port, Reason]),
                halt(1)
        end
    catch
        _:Error ->
            io:format("Error: ~p~n", [Error]),
            usage()
    end;
main(_) ->
    usage().



usage() ->
    io:format("Usage: server <port> <file> <send|sendfile>~n"),
    halt(1).


fstat_monitor(File) ->
    receive
        blocked ->
            io:format("fstat on ~p is blocked~n", [File]),
            fstat_monitor(File);
        stop ->
            ok
    end.


fstat(File) ->
    Pid = spawn(node(), fun() -> fstat_monitor(File) end),
    {ok, TRef} = timer:send_interval(5000, Pid, blocked),
    file:read_file_info(File),
    timer:cancel(TRef),
    Pid ! stop,
    io:format("fstat on ~p returned~n", [File]),
    timer:sleep(2000),
    fstat(File).


do_accept(LSock, Opts) ->
    case gen_tcp:accept(LSock) of
        {ok, Sock} ->
            {ok, {PeerName, PeerPort}} = inet:peername(Sock),
            io:format("~p Connection accepted from ~s:~p~n",
                      [Sock, inet_parse:ntoa(PeerName), PeerPort]),
            Fun = proplists:get_value(function, Opts),
            case Fun of
                send ->
                    Pid = spawn(node(), fun() -> send(Sock, Opts) end),
                    gen_tcp:controlling_process(Sock, Pid),
                    ok;
                sendfile ->
                    Pid = spawn(node(), fun() -> sendfile(Sock, Opts) end),
                    gen_tcp:controlling_process(Sock, Pid),
                    ok;
                _ ->
                    io:format("undefined function ~p~n", [Fun]),
                    gen_tcp:close(Sock),
                    gen_tcp:close(LSock),
                    halt(1)
            end,
            do_accept(LSock, Opts);
        {error, Reason} ->
            io:format("Failed to accept connection: ~p~n", [Reason]),
            gen_tcp:close(LSock),
            halt(1)
    end.

send(Sock, Opts) ->
    File = proplists:get_value(file, Opts),
    io:format("~p Send ~p using gen_tcp:send~n", [Sock, File]),
    {ok, #file_info{size = Size}} = file:read_file_info(File),
    {ok, Fd} =  file:open(File, [read, binary, raw]),
    do_send(Sock, Fd, Size),
    file:close(Fd),
    gen_tcp:close(Sock),
    io:format("~p Connection closed~n", [Sock]),
    ok.

do_send(Sock, Fd, Count) when Count < 10240 ->
    {ok, Data} = file:read(Fd, Count),
    case gen_tcp:send(Sock, Data) of
        ok ->
            io:format("~p ~p bytes sent to client~n", [Sock, Count]);
        {error, Reason} ->
            io:format("~p failed to send file: ~p~n", [Sock, Reason])
    end;
do_send(Sock, Fd, Count) ->
    {ok, Data} = file:read(Fd, 10240),
    case gen_tcp:send(Sock, Data) of
        ok ->
            io:format("~p ~p bytes sent to client~n", [Sock, 10240]),
            do_send(Sock, Fd, Count - 10240);
        {error, Reason} ->
            io:format("~p failed to send file: ~p~n", [Sock, Reason])
    end.


sendfile(Sock, Opts) ->
    File = proplists:get_value(file, Opts),
    io:format("~p Send ~p using file:sendfile~n", [Sock, File]),
    case file:sendfile(File, Sock) of
        {ok, BytesSend} ->
            io:format("~p ~p bytes sent to client~n", [Sock, BytesSend]);
        {error, Reason} ->
            io:format("~p failed to send file: ~p~n", [Sock, Reason])
    end,
    gen_tcp:close(Sock),
    io:format("~p Connection closed~n", [Sock]),
    ok.


-------------- next part --------------
#!/usr/bin/env escript
%% -*- erlang -*-
%%! -smp enable -sname slow_client

main([Ip, Port0]) ->
    try
        Port  = list_to_integer(Port0),
        Opts  = [
                 binary,
                 {packet, 0},
                 {active, once},
                 {recbuf, 1024}
                ],
        case gen_tcp:connect(Ip, Port, Opts) of
            {ok, Sock} ->
                io:format("~p Client connected on ~s:~p~n", [Sock, Ip, Port]),
                do_receive(Sock, 0, now());
            {error, Reason} ->
                io:format("Failed to connect on ~s:~p: ~p~n",
                          [Ip, Port, Reason]),
                halt(1)
        end
    catch
        _:Error ->
            io:format("Error: ~p~n", [Error]),
            usage()
    end;
main(_) ->
    usage().


usage() ->
    io:format("Usage: client <ip> <port>~n").


do_receive(Sock, NRead, StartTime) ->
    receive
        {tcp, Sock, Data} ->
            NRead1 = NRead + size(Data),
            Rate = (1000 * NRead1) / timer:now_diff(now(), StartTime),
            io:format("~p ~p bytes received (avg rate: ~.3f KB/s)~n",
                      [Sock, NRead1, Rate]),
            timer:sleep(10000),
            inet:setopts(Sock, [{active, once}]),
            do_receive(Sock, NRead1, StartTime);
        {tcp_closed, Sock} ->
            io:format("~p Connection closed~n", [Sock])
    end.
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 259 bytes
Desc: OpenPGP digital signature
URL: <http://erlang.org/pipermail/erlang-bugs/attachments/20131021/83510105/attachment.bin>