[erlang-questions] compile module from string, containing macro definitions

[Mats Cronqvist]

Chris Newcombe wrote:
> I'm using this code from Ulf Wiger to compile a module from a string:

   bad mistake... (hi ulf!)

 > However, when I test it with code containing a macro definition I get an
 > error.

   "it hurts when i do this..." :>

   as it were, i too have needed to compile a string that contains macros.

   the attached code works without any hacking of OTP. it will compile a string 
with macros. be aware that the macro handling is very limited (but hopefully 
easy to extend).

   e.g.

 > complainer:string("-module(x).\n-export([go/0]).\n-define(X,\"that 
sucks\").\ngo()->[erlang,has_macros,?X].\n").
{module,x}
 > x:go().
[erlang,has_macros,"that sucks"]

   mats

%%----------------------------------------------------------------
%%% File        : complainer.erl
%%% Author      : Mats Cronqvist <locmacr@REDACTED>
%%% Created     : 10 Jul 2006
%%% Description : takes a string that should be an erlang module.
%%%               scans, does macro substitution,
%%%               parses, compiles and loads.
%%%               will also make popcorn.
%%%----------------------------------------------------------------

-module(complainer).
-author('Mats Cronqvist').

-export([string/1]).

-import(lists,[reverse/1,keyreplace/4]).

string(Text) ->
     Forms = scan_and_parse(Text,1,[],dict:new()),
     {ok,Mod,Bin} = compile:forms(Forms),
     code:load_binary(Mod,"generated",Bin).

%%%## 'scan_and_parse'
%%%
%%% basically we call the OTP scanner and parser (erl_scan and
%%% erl_parse) line-by-line, but check each scanned line for (or
%%% definitions of) macros before parsing.

scan_and_parse([],_Line,Forms,_MacroDict) -> reverse(Forms);
scan_and_parse(Text,Line,Forms,MacroDict) ->
     case scanner(Text,Line,MacroDict) of
	{macro,NLine,Cont,NMacroDict} ->
	    scan_and_parse(Cont,NLine,Forms,NMacroDict);
	{tokens,NLine,Cont,Toks} ->
	    {ok,Form} = erl_parse:parse_form(Toks),
	    scan_and_parse(Cont,NLine,[Form|Forms],MacroDict)
     end.

scanner(Text,Line,MacroDict) ->
     {done,{ok,Toks,NLine},Cont} = erl_scan:tokens([],Text,Line),
     case pre_proc(Toks,MacroDict) of
	{macro,NMacroDict} ->
	    {macro,NLine,Cont,NMacroDict};
	{tokens,NToks} ->
	    {tokens,NLine,Cont,NToks}
     end.

%%%## 'pre-proc'
%%%
%%% have to implement a subset of the pre-processor, since epp insists
%%% on running on a file.
%%% only handles 2 cases;
%% -define(MACRO,"string").
%% -define(MACRO(VAR1,VARN),{stuff,VAR1,more,stuff,VARN,extra,stuff}).

pre_proc([{'-',_},{atom,_,define},{'(',_},{_,_,Name}|DefToks],MacroDict) ->
     false = dict:is_key(Name,MacroDict),
     case DefToks of
	[{',',_}|Macro] ->
	    {macro,dict:store(Name,{[],macro_wo_vars(Macro,[])},MacroDict)};
	[{'(',_}|Macro] ->
	    {macro,dict:store(Name,macro_w_vars(Macro,[]),MacroDict)}
     end;
pre_proc(Toks,MacroDict) ->
     {tokens,subst_macros(Toks,MacroDict,[])}.

macro_w_vars([{')',_},{',',_}|Toks],Vars) ->
     {reverse(Vars),macro_wo_vars(Toks,[])};
macro_w_vars([{var,_,Var}|Toks],Vars) ->
     macro_w_vars(Toks,[Var|Vars]);
macro_w_vars([{',',_},{var,_,Var}|Toks],Vars) ->
     macro_w_vars(Toks,[Var|Vars]).

macro_wo_vars([{')',_},{dot,_}],Val) -> Val;
macro_wo_vars([H|T],Val) -> macro_wo_vars(T,[H|Val]).

subst_macros([{'?',_},{_,_,Name},{'(',_}|Toks],MacroDict,O) ->
     {NToks,Vars} = subst_macros_get_vars(Toks,[]),
     Macro = dict:fetch(Name,MacroDict),
     subst_macros(NToks,MacroDict,subst_macros_put_vars(Macro,Vars)++O);
subst_macros([{'?',_},{_,_,Name}|Toks],MacroDict,O) ->
     Macro = dict:fetch(Name,MacroDict),
     subst_macros(Toks,MacroDict,subst_macros_put_vars(Macro,[])++O);
subst_macros([H|T],MacroDict,O) -> subst_macros(T,MacroDict,[H|O]);
subst_macros([],_MacroDict,O) -> reverse(O).


subst_macros_get_vars([{')',_}|Toks],O) ->
     {Toks,reverse(O)};
subst_macros_get_vars([{',',_},{var,_,Var}|Toks],O) ->
     subst_macros_get_vars(Toks,[Var|O]);
subst_macros_get_vars([{var,_,Var}|Toks],O) ->
     subst_macros_get_vars(Toks,[Var|O]).

subst_macros_put_vars({[],Macro},[]) -> Macro;
subst_macros_put_vars({[MVar|MVars],Macro},[Var|Vars]) ->
     NMacro = keyreplace(MVar,3,Macro,{var,1,Var}),
     subst_macros_put_vars({MVars,NMacro},Vars).