Shock horror

[Joe Armstrong]

   Complexity Shock Horror ....

   I have found to my horror and amazement that that our beloved
Erlang system is far more complex than it should be ...

   ... it all started  when I started reading systools.erl ...

   Warning -  timid souls  should not read  on for  this is a  tale of
horror that will chill your very entrails and make you doubt your sanity

   ....

   (scroll the page for more)




























   I wanted to change 

	systools:script2boot/1

   Now this function calls 

	systools_lib:file_term2binary/2

   and I could not understand why the code for this had been placed in
a *different* module - surely *nobody* would want to call
systools_lib:file_term2binary/2 - it's so obscure that nobody would
ever think to use it. And it's not even documented (systools_lib) has
no man pages - so the chances that anybody would ever even think of
using this function are minuscule.

   I thought to myself - "I'll do a Xref analysis to find out if
anybody calls systools_lib:file_term2binary/2" - this turned out to be
more difficult than I had imagined - xref once small and easy to use has
grown in complexity since I last used it to the point where it is almost
(but not quite) impossible to use. 

  <<xrefs manual pages seem also to have degenerated to the point of
unusability - but *that* is another question>>

  After a few false starts and guesses to what xref did I was able to
analyse the system to see who called "systools_lib:file_term2binary/2" -
and my intuition was correct:

  systools_lib:file_term2binary/2 is only called from systools.erl

  Now I might be completely "out and cycling" here (as my Swedish
friends would say) - but IMMVHO (Most very :-) if it is the case
that only one other module calls an exported function then the
function could be MOVED to the place from whence it was called.

  A case of

	"Whither wert tho called oh systools_lib:file_term2binary/2?

	"From only systools.erl"

        "Then go ye thence"

  So I thought to myself - How many exported functions are "singletons"
(by that I means functions which are only called from one external
module) - To answer this I wrote a little program...

  During the writing, thereof, I discovered that there were not only a
large number of singletons (I had hoped that I had stumbled across the
only one) but also a large number of "nollitons" - exported functions
that *nobody* calls.

  For your edification and delight, the numbers are:

   #exported functions = 8148
   4632 functions do not need to be exported (nollitons)
   2225 functions can be moved               (singletons)
   1291 functions cannot be moved

 Pretty stinky stuff, ej wot.

 IMHO every time a nolliton or singleton is used the system become
more difficult to understand and more difficult to structure and more
difficult optimise - "module at a time" compilers do not like external
functions - and thinking up sensible names for them is a pain in the
nether regions.

 Even worse I don't even know if we can remove the buggers - perhaps
(horrors) some application actually calls an undocumented nolliton -
removing (say) all nollitons might awaken the the great God of
backwards incompatibility - their might be much gnashing and grinding
of teeth and wailing and crying if we did this ...

 Before the age utopic bliss is achieved (thank you Thomas) we could
perhaps check to see if any of the BIG applications call the nasty
little nollitons and singletons ...

 Do they?
 
 Run the enclosed program to see (adding you own own application
modules to the set of stuff that is analysed)

 /Joe

  PS cannot anything  be done about this, apart  from a total re-write
using Richards heirarchical module system  (which I guess we should do
at the same time :-)










-------------- next part --------------
-module(anal).

-compile(export_all).
-import(lists, [foldl/3, foreach/2, map/2, member/2, sort/1]).


%% anal:go(all)
%%  runs the analysis

libs(small) ->
    [stdlib, kernel, sasl];
libs(all) ->
    [erl_interface,mnemosyne,sasl,appmon,eva,mnesia,snmp,asn1,
     gs,mnesia_session,ssl,odbc,stdlib,compiler,ic,toolbar,
     inets,os_mon,tools,parsetools,tv,debugger,kernel,pman,
     runtime_tools].

go(Type) ->
    {ok, P} = xref:start(a),
    add_libs(P, libs(Type)),
    R = xref:analyse(P, {use, {code, lib_dir, 1}}, []),
    io:format("R=~p~n",[R]),
    R1 = xref:analyse(P, {use, {systools_lib, file_term2binary, 2}}, []),
    
    Mods = map(fun({I,_}) -> I end, xref:info(P, modules)),
    io:format("Mods=~p~n",[Mods]),
    L1 = foldl(fun(Mod, A) ->
		       Exports = Mod:module_info(exports),
		       Exports1 = remove_module_info(Exports),
		       foldl(fun({Func, Arity}, A0) ->
				     MFA = {Mod, Func, Arity},
				     case xref:analyse(P,{use, MFA}) of
					 {ok, L} ->
					     Tup = {MFA, L},
					     [Tup|A0];
					 Other ->
					     io:format("uugh:~p~n",[Other]),
					     A0
				     end
			     end, A, Exports1)
	       end, [], Mods),
    io:format("L1=~p~n",[L1]),
    L2 = compute_complexity(L1),
    {ok, S} = file:open("anal_result.tmp", write),
    io:format(S, "~p~n",[L2]),
    file:close(S),
    xref:stop(P).


%% The completity of aan exported function =
%%    #Emods = the number of external modules
%%             that call the function


%% We want to compute the following
%%  The number of functions that should not be exported
%%  = (the number of fucntions that should not be exported)

compute_complexity(L) ->
    %% Each exported function counts 1
    Cinit = length(L),
    io:format("Current complexity= ~p~n", [Cinit]),
    L1 = map(fun({{M,F,A}, La}) ->
		     {{M,F,A}, external_exports(M, La, [])}
	     end, L),
    %% The function which do not need to be exported
    L2 = [MFA||{MFA, []} <- L1],
    io:format("~w functions do not need to be exported~n", [length(L2)]),
    L3 = [MFA||{MFA, [X]} <- L1],
    io:format("~w functions can be moved~n", [length(L3)]),
    L4 = [MFA||{MFA, X} <- L1, length(X) > 1],
    io:format("~w functions cannot be moved~n", [length(L4)]),
    L5 = sort(map(fun({MFA,E}) ->
			  {length(E), MFA, E}
		  end, L1)).

remove_module_info([{module_info,_}|T]) -> remove_module_info(T);
remove_module_info([H|T]) -> [H|remove_module_info(T)];
remove_module_info([]) -> [].
			
external_exports(M, [{M,F,A}|T], External) ->
    external_exports(M, T, External);
external_exports(M1, [{M2,F,A}|T], External) ->
    case member(M2, External) of
	true ->
	    external_exports(M1, T, External);
	false ->
	    external_exports(M1, T, [M2|External])
    end;
external_exports(M, [], E) ->
    E.

complexity(Mod, [{Mod,_,_}|T]) -> complexity(Mod, T);
complexity(Mod, [H|T])         -> [H|complexity(Mod, T)];
complexity(Mod, [])            -> [].

add_libs(P, L) ->
    foreach(fun(I) ->
		    Dir = code:lib_dir(I) ++ "/ebin",
		    io:format("adding:~p~n", [Dir]),
		    X = xref:add_directory(P, Dir, [{verbose,false}]),    
		    io:format("X=~p~n", [X])
	    end, L).