[erlang-questions] Inter-node communication bottleneck

[Jihyun Yu]

Hi,

I couldn't spend much time after last update. It seems that using
multiple TCP sockets is not that scalable as well. I ran a test with
following configurations.

 - 12 cores, 24 threads server (Intel L5640 x 2)
 - Two Erlang VM on two docker containers
 - Connects with 12 TCP connections and send/recv packets
 - [{delay_send,true}, {high_watermark,1024000}]

With single integer term, total bandwidth is capped on about ~10MB/s.
With single binary term <<"hello">>, total bandwidth is capped on about
20~30MB/s. Two Erlang VM consume almost all CPU resources (2400% total),
so bandwidth per CPU usage is much worse than Erlang messaging case. I
didn't run perf yet, so I don't know where is an bottleneck.

I attached test source code so you can reproduce the result. Please tell
me if there is an error on configurations/test codes/...


[1] http://www.erlang.org/doc/man/inet.html

On Mon, Sep 08, 2014 at 03:59:04AM -0700, Dror Mein wrote:
> I encountered the same problem. any updates or new successes? the binary_to_term over tcp sounds promising.
-------------- next part --------------
-module(test_tcp).

-export([run_tcp/3, recv_tcp/3, send_tcp/4]).

-define(SOCKET_OPTS, [{active, false}, binary, {packet, 4}]).

run_tcp(Port, P, N) ->
    spawn_link(fun() ->
        recv_tcp(Port, P, N)
    end),
    send_tcp(localhost, 10000, P, N).

recv_tcp(Port, P, N) ->
    {ok, ListenSocket} = gen_tcp:listen(Port, ?SOCKET_OPTS),
    lists:foreach(fun(Idx) ->
        {ok, Socket} = gen_tcp:accept(ListenSocket),
        spawn_monitor(fun() ->
            io:format("receiving ~p/~p~n", [Idx, P]),
            recv_n(Socket, N),
            io:format("receiving ~p/~p done~n", [Idx, P]),
            ok = gen_tcp:close(Socket)
        end)
    end, lists:seq(1, P)),
    waitdown(P),
    io:format("closing listen socket~n"),
    ok = gen_tcp:close(ListenSocket).

recv_n(_Socket, 0) -> ok;
recv_n(Socket, N) ->
    {ok, _Data} = gen_tcp:recv(Socket, 0),
    recv_n(Socket, N-1).

send_tcp(Host, Port, P, N) ->
    lists:foreach(fun(Idx) ->
        spawn_monitor(fun() ->
            {ok, Socket} = gen_tcp:connect(Host, Port, ?SOCKET_OPTS),
            io:format("sending ~p/~p~n", [Idx, P]),
            send_n(Socket, N),
            io:format("sending ~p/~p done~n", [Idx, P]),
            ok = gen_tcp:close(Socket)
        end)
    end, lists:seq(1, P)),
    waitdown(P).

send_n(_, 0) -> ok;
send_n(Socket, N) ->
    gen_tcp:send(Socket, term_to_binary(N)),
    send_n(Socket, N-1).

waitdown(0) -> ok;
waitdown(Count) ->
    receive
        {'DOWN', _, process, _, normal} ->
            waitdown(Count-1)
    end.