gen_tcp very slow to fetch data

zabrane Mikael <>
Mon Nov 16 17:17:32 CET 2009


Hi List !

New to Erlang, I'm trying to implement a simple URL fetcher.
Here's my code (please, feel free to correct it if you find any bug or know
a better approach):

8-----8-----8-----8-----8-----8-----8-----8-----8-----8-----8-----8-----8----
-module(fetch).

-export([url/1]).

-define(TIMEOUT,    7000).
-define(TCP_OPTS,   [binary, {packet, raw}, {nodelay, true},
                     {active, true}]).

url(Url) ->
    {ok, _Tag, Host, Port} = split_url(Url),

    Hdrs = [],
    Request = ["GET ", Url, " HTTP/1.1\r\n", Hdrs, "\r\n\r\n"],

    case catch gen_tcp:connect(Host, Port, ?TCP_OPTS) of
      {'EXIT', Why} ->
            {error, {socket_exit, Why}};
        {error, Why} ->
            {error, {socket_error, Why}};
        {ok, Socket} ->
            gen_tcp:send(Socket, list_to_binary(Request)),
            recv(Socket, list_to_binary([]))
    end.

recv(Socket, Bin) ->
    receive
        {tcp, Socket, B} ->
            io:format(".", []),
            recv(Socket, concat_binary([Bin, B]));
        {tcp_closed, Socket} ->
            {ok, Bin};
        Other ->
            {error, {socket, Other}}
after
    ?TIMEOUT ->
            {error, {socket, timeout}}
    end.


split_url([$h,$t,$t,$p,$:,$/,$/|T]) ->  split_url(http, T);
split_url(_X)                       ->  {error, split_url}.

split_url(Tag, X) ->
    case string:chr(X, $:) of
        0 ->
            Port = 80,
            case string:chr(X,$/) of
                0 ->
                    {ok, Tag, X, Port};
                N ->
                    Site = string:substr(X,1,N-1),
                    {ok, Tag, Site, Port}
            end;
        N1 ->
            case string:chr(X,$/) of
                0 ->
                    error;
                N2 ->
                    PortStr = string:substr(X,N1+1, N2-N1-1),
                    case catch list_to_integer(PortStr) of
                        {'EXIT', _} ->
                            {error, port_number};
                        Port ->
                            Site = string:substr(X,1,N1-1),
                            {ok, Tag, Site, Port}
                    end
            end
    end.

8-----8-----8-----8-----8-----8-----8-----8-----8-----8-----8-----8-----8------

When testing it, the receiving socket gets very very slow:
$ erl
1> c(fetch).
2> Bin = fetch:url("http://www.google.com").
......{error,{socket,timeout}}

Am I missing something?
What I like to get at the end is a very fast fetcher. Any hint?

Regards
Zabrane


More information about the erlang-questions mailing list