[erlang-questions] disk tests

[Joe Armstrong]

Hello,

I've written a little program to test writing to disk - I need to
create a number of large random access files
and was running some tests when I ran into problems. As I write files
to disk the erlang process grows
and eventually I hit some kind of memory limits and the system slows to a crawl.

It also has this behavior for sequential writes to disk. Also the data
rate to disk is not as fast as I would have
expected.

Is this a bug or a feature?

 Here's a program that reproduces this. When I run disk_test:test() my
system slows to a crawl.
Can anybody explain what is happening here?

Best

/Joe

-module(disk_test).

%% Time-stamp: <2008-10-09 15:16:16 joe>

-import(lists, [min/1, duplicate/2]).

-compile(export_all).

%% This program was supposed to test how fast I could write to disk
%% but when I run it, it virtually stops the system.
%% I create a 2 Meg file filled with zeros, then a 4 Meg file
%% then a 6 Meg file and so on.

%% By the time I get to creating a 40 MByte file the system has slowed to
%% a crawl. The total data written at this point is around 500 Meg.
%% If I watch the Memory usage my Erlang process has run up to 2GB of
%% memory and the entire system crawls along. I then have to kill Erlang.

%% The first few lines of output when I run this are as follows:

%% > disk_test:test().
%% Creating a 2 Megabyte file
%% {created,2,megs,in,0.225377,secs,8.874019975418966,
%%          megaBytesPerSec}.
%% Creating a 4 Megabyte file
%% {created,6,megs,in,0.49773,secs,12.054728467241276,
%%          megaBytesPerSec}.
%% Creating a 6 Megabyte file
%% {created,12,megs,in,0.930026,secs,12.902865081191278,
%%          megaBytesPerSec}.
%% Creating a 8 Megabyte file
%% {created,20,megs,in,1.461848,secs,13.681312968242937,
%%          megaBytesPerSec}.
%% Creating a 10 Megabyte file
%% {created,30,megs,in,2.213844,secs,13.551090320727207,
%%          megaBytesPerSec}.
%% Creating a 12 Megabyte file
%% {created,42,megs,in,2.926186,secs,14.353154584158355,
%%          megaBytesPerSec}.

%% I'm managing to write about 12 MBytes/sec
%% I suspect the disk is capable of about 70 MB/sec
%% (just a guess)

%% What's happening here? - why does the process size grow so large?

test() ->
    make_file_seq(2,2,now_micros()).

make_file_seq(N,_,_) when N > 60 ->
    true;
make_file_seq(N,Tot,Tstart) ->
    make_file_seq0(N),
    Secs = duration(Tstart),
    Rate = Tot/Secs,
    io:format("~p.~n",[{created, Tot,
			megs,in, Secs,secs, Rate,megaBytesPerSec}]),
    make_file_seq(N+2, Tot+N+2, Tstart).

duration(Tstart) ->
    (now_micros() - Tstart)/1000000.

now_micros() ->
    {Mega,Sec,Micro} = erlang:now(),
    (Mega*1000000 + Sec)*1000000 + Micro.

make_file_seq0(N) ->
    io:format("Creating a ~p Megabyte file~n",[N]),
    File = "tmp",
    file:delete(File),
    {ok, S} = file:open(File, [write]),
    Size = N * 1000000,
    zero_file_seqwrite(S, Size),
    file:close(S),
    Size = filelib:file_size(File). %% just checking :-)

%% zero_file_seqwrite_seqwrite(FileHandle, Len)
%%    writes Len zeros to a file

zero_file_seqwrite(S, Len) ->
    N = min([Len, 4096]),
    B = list_to_binary(duplicate(N, 0)),
    zero_file_seqwrite(S, Len, B, N).

zero_file_seqwrite(S, Len, B, Size) when Size > Len ->
    file:write(S, [B]),
    zero_file_seqwrite(S, Len-Size, B, Size);
zero_file_seqwrite(S, Len, _, _) ->
    B = list_to_binary(duplicate(Len, 0)),
    file:write(S, [B]).