[erlang-questions] Unstable erlang compared to java or perl

Petter Egesund <>
Sun Nov 7 16:39:43 CET 2010


Sure :-)

The most important part of my main loop works like this:

Spawn up to a given number of threads. When a thread dies it sends a
msg back to the main loop, so I can spawn a new one. And so on... The
problem, as it seems to me, is that ets-garbage from dead threads
never are collected?

I have also tried to do this from my initial call:
erlang:system_flag(fullsweep_after, 0), but without any change.

Code:

parse_dir_loop(ThreadCounter, MaxThreads, Dir, OldSpawnFiles) ->
    io:fwrite("Number of processes: ~p ~n", [length(processes())]),
    ets:i(),
    ThreadCounterNew = case ThreadCounter < MaxThreads of
               true -> {ok, RemainingFiles} = file:list_dir(Dir),
                   case RemainingFiles of
                       [] -> SpawnFiles = [], ThreadCounter;
                       Files -> SpawnFilesNum = min(MaxThreads -
ThreadCounter, length(Files -- OldSpawnFiles)),
                        SpawnFiles = lists:sublist(lists:sort(Files --
OldSpawnFiles), 1, SpawnFilesNum),
                        Receiver = self(),
                        lists:foreach(fun(File) -> spawn_link(fun() ->
                                                  parse_file(Dir ++ File),
                                                  %% Receiver ! {done, File},
                                                  exit({kill_me, File})
                                              end)
                                  end, SpawnFiles),
                        ThreadCounter + SpawnFilesNum
                   end;
               false -> SpawnFiles = [], ThreadCounter
               end,
    NewSpawnFiles = OldSpawnFiles ++ SpawnFiles,
    receive
    {done, File} ->
        io:fwrite("Done with a file ~n"),
        parse_dir_loop(ThreadCounterNew - 1, MaxThreads, Dir,
NewSpawnFiles -- File);
    {'EXIT', Pid, {kill_me, File}} ->
        io:fwrite("Should be dead by now: ~p ~n", [Pid]),

        parse_dir_loop(ThreadCounterNew - 1, MaxThreads, Dir,
NewSpawnFiles -- File);
    {'EXIT', Pid, normal} ->
        io:fwrite("Should be dead by now: ~p ~n", [Pid]),
        garbage_collect(Pid),
        parse_dir_loop(ThreadCounter, MaxThreads, Dir, NewSpawnFiles);
    Signal -> io:fwrite("Got unknown signal in parse_dir_loop ~p~n", [Signal])
    after 5000 -> case file:list_dir(Dir) of
              {ok, []} -> io:fwrite("Parsing directory is empty - job
done: ~p~n", [Dir]);
              {ok, _} -> parse_dir_loop(ThreadCounterNew, MaxThreads,
Dir, NewSpawnFiles)
          end
    end.


parse_dir(ThreadCounter, MaxThreads, Dir) ->
    %% probably fix path for Windows?!
    parse_dir_loop(ThreadCounter, MaxThreads, Dir, []).


On Sun, Nov 7, 2010 at 4:27 PM, Bengt Kleberg
<> wrote:
> Greetings,
>
> Could you include the program, or preferably a small subset of it, that
> runs out of memory?
>
>
> bengt
>
> On Sun, 2010-11-07 at 15:49 +0100, Petter Egesund wrote:
>> Hi, I have a small program with lots of memory-updates which I try to
>> run in Erlang.
>>
>> The same algorithm works fine in both Java and Perl, but fails in
>> Erlang because the program runs out of memory - and I can not figure
>> out why. Frustrating, as my Erlang-versjon seems to be the easiest to
>> scale as well as being the most readable.
>>
>> The program is threaded and each thread writes to a ets-table which is
>> created at the beginning of the thread. When the thread dies I try to
>> do a ets:delete(Table), like described in the manual, but the memory
>> used by the thread never seems to be released.
>>
>> Some facts:
>>
>> - The memory usage of each thread is rather constant. This is
>> confirmed when I use ets:i() to show info about memory usage.
>> - The number of threads are constant - confirmed by both running top
>> and writing out the number of threads regularly. When a thread dies, I
>> create a new one.
>> - I have tried to end the thread by sending a exit-signal as the last
>> statement. This helps some, but does not solve the leak.
>> - I put small lists of size 3-4 integers into the ets as values, the
>> keys are list of same size as well.
>> - I garbage-collect each thread before it dies, as well as doing
>> regular global garbage-collects. No help.
>> - Information from ets:i() about memory when I sum usage by each
>> thread, is much lower than stated by memory() when i run
>> erlang:memory(). This might indicate something? Does not seem logical
>> to me, at least.
>> - Info from erlang:memory is about half of what top/the os tells.
>> - I am running on ubuntu, 64-bit, 14A but I have tried 14B as well.
>>
>> Any clues? Dump from ets:i() and erlang:memory() is like below.
>>
>> Cheers,
>>
>> Petter
>>
>> --- dump ---
>>
>> eNumber of processes: 27
>> ets:i():
>>  id              name              type  size   mem      owner
>>  ----------------------------------------------------------------------------
>>  13              code              set   261    10692    code_server
>>  4110            code_names        set   58     7804     code_server
>>  6746271765      the_synapses      ordered_set 5425194 113336012 <0.47.0>
>>  7022018584      the_synapses      ordered_set 15143493 310909950 <0.48.0>
>>  7774416922      the_synapses      ordered_set 8794649 182005810 <0.49.0>
>>  ac_tab          ac_tab            set   6      848      application_controller
>>  file_io_servers file_io_servers   set   0      302      file_server_2
>>  global_locks    global_locks      set   0      302      global_name_server
>>  global_names    global_names      set   0      302      global_name_server
>>  global_names_ext global_names_ext  set   0      302      global_name_server
>>  global_pid_ids  global_pid_ids    bag   0      302      global_name_server
>>  global_pid_names global_pid_names  bag   0      302      global_name_server
>>  inet_cache      inet_cache        bag   0      302      inet_db
>>  inet_db         inet_db           set   29     571      inet_db
>>  inet_hosts_byaddr inet_hosts_byaddr bag   0      302      inet_db
>>  inet_hosts_byname inet_hosts_byname bag   0      302      inet_db
>>  inet_hosts_file_byaddr inet_hosts_file_byaddr bag   0      302      inet_db
>>  inet_hosts_file_byname inet_hosts_file_byname bag   0      302      inet_db
>>  neurone_counter neurone_counter   set   258394 1846182  entity_server
>>  neurone_group_counter neurone_group_counter set   6      344
>> entity_group_server
>>  neurone_group_name neurone_group_name set   6      426      entity_group_server
>>  neurone_group_name_reverse neurone_group_name_reverse set   6
>> 426      entity_group_server
>>  neurone_name    neurone_name      set   258394 11824602 entity_server
>>  neurone_name_reverse neurone_name_reverse set   258394 11824602 entity_server
>> memory():       [{total,5568669792},
>>                    {processes,1138936},
>>                    {processes_used,1128120},
>>                    {system,5567530856},
>>                    {atom,349769},
>>                    {atom_used,336605},
>>                    {binary,82704},
>>                    {code,3046365},
>>                    {ets,5562163256}]
>>
>> ________________________________________________________________
>> erlang-questions (at) erlang.org mailing list.
>> See http://www.erlang.org/faq.html
>> To unsubscribe; mailto:
>>
>
>
> ________________________________________________________________
> erlang-questions (at) erlang.org mailing list.
> See http://www.erlang.org/faq.html
> To unsubscribe; mailto:
>
>


More information about the erlang-questions mailing list