Change node name in mnesia

Vance Shipley vances@REDACTED
Thu Oct 21 00:41:57 CEST 2010


Dan,

I am able to reproduce the problem with the attached code and 
the procedure detailed below.  In a nutshell the process outlined
in The Mnesia User's Guide for renaming a database node in a
backup file results in a corrupt database.

     http://erlang.org/doc/apps/mnesia/Mnesia_chap7.html#id74479

I used the attached test code, derived from the above example, to
create a sample database, rename the node name from long form to
short and verify it's integrity.

Start a node with a long node name:

   $ erl -name long
   Eshell V5.8.1  (abort with ^G)
   (long@REDACTED)1> rr("db_transform.erl").
   [acc,bar,baz,foo]

Build an example data base:

   (long@REDACTED)2> db_transform:install([node()], 4000, 500, 25).
   {ok,#acc{foo = 3994,bar = 500,baz = 25}}

Back up the example database into the file "long":

   (long@REDACTED)3> mnesia:backup(long).
   ok

Start a node with a short node name:

   $ erl -sname short
   Eshell V5.8.1  (abort with ^G)
   (short@REDACTED)1>  rr("db_transform.erl").
   [acc,bar,baz,foo]

Change the node name in the backup file "long" and create backup "short":

   (short@REDACTED)2> db_transform:rename('long@REDACTED', 'short@REDACTED', long, short).
   {ok,#acc{foo = 3994,bar = 500,baz = 25}}

Create a new database from the backup "short":

   (short@REDACTED)3> mnesia:create_schema([node()]).
   ok
   (short@REDACTED)4> mnesia:start().
   ok
   (short@REDACTED)5> mnesia:restore(short, [{default_op, recreate_tables}]).
   {atomic,[foo,baz,bar]}

Test the database integrity:

   (short@REDACTED)6> db_transform:verify([foo, bar, baz]).
   #acc{foo = 3994,bar = 500,baz = 25}

The above proves the backup, transform and restore process works.
So far so good.  Now below is where the problem can be reproduced.

   (short@REDACTED)7> q().
   ok
	$ erl -sname short
	(short@REDACTED)1> mnesia:start().
   ok
   (short@REDACTED)2> db_transform:verify([foo, bar, baz]).
   ** exception exit: {function_clause,[{db_transform,'-verify/2-fun-0-',
                                                      [{log_header,dcl_log,"1.0","4.4.15",'short@REDACTED',
                                                                   {1287,613266,224499}},
                                                       1899]},
                                        {lists,foldl,3},
                                        {mnesia,do_foldl,8},
                                        {mnesia,foldl,6},
                                        {mnesia_tm,apply_fun,3},
                                        {mnesia_tm,execute_transaction,5},
                                        {db_transform,count,2},
                                        {db_transform,verify,2}]}
        in function  db_transform:count/2
        in call from db_transform:verify/2

Restarting the node is what triggers the problem.

   (short@REDACTED)3> ets:i(baz).
   <1   > {baz,711590,
        <<76,16,157,97,48,225,132,80,36,196,3,199,194,239  ...
   <2   > {baz,411561,
        <<173,175,232,135,28,118,233,210,111,49,169,19,88  ...
   <3   > {baz,909239,
        <<135,214,14,32,8,121,144,64,196,35,186,167,63,24  ...
   <4   > {baz,463433,
        <<66,156,188,32,49,157,54,160,133,241,196,78,22,2  ...
   <5   > {baz,997452,
        <<248,253,234,156,76,130,48,212,89,197,186,210,21  ...
   <6   > {baz,797390,
        <<25,1,27,228,30,234,2,147,80,80,167,58,230,54,11  ...
   <7   > {baz,879889,
        <<249,45,35,209,179,173,217,158,208,227,220,67,20  ...
   <8   > {baz,604804,
        <<120,75,101,145,118,116,39,218,204,246,198,162,3  ...
   <9   > {baz,807967,
        <<58,209,110,188,212,128,136,15,133,56,166,13,30,  ...
   <10  > {baz,388388,
        <<143,53,248,42,115,146,218,222,246,67,10,173,185  ...
   <11  > {baz,890638,
        <<11,137,19,231,155,97,254,94,79,68,94,82,156,190  ...
   <12  > {baz,865475,
        <<169,173,174,222,235,89,94,90,11,192,50,72,37,43  ...
   <13  > {baz,595736,
        <<43,98,245,177,0,79,54,225,36,201,24,188,195,240  ...
   <14  > {log_header,dcl_log,"1.0","4.4.15",'short@REDACTED',
            ...
   <15  > {baz,771466,
        <<39,59,205,76,71,40,80,85,65,229,224,198,109,132  ...
   <16  > {baz,834821,
        <<76,65,143,50,217,159,155,39,129,56,67,59,73,95,  ...
   <17  > {baz,534651,
        <<72,192,141,91,154,171,201,126,135,7,213,143,13,  ...
   <18  > {baz,551117,
        <<85,185,77,144,114,113,203,93,206,29,122,106,52,  ...
   <19  > {baz,224818,
        <<120,27,23,224,251,51,254,69,215,104,11,93,135,5  ...
   <20  > {baz,358442,
        <<255,165,10,214,6,121,98,31,231,82,52,194,27,94,  ...
   <21  > {baz,382522,
        <<188,190,108,27,51,81,186,175,215,95,168,203,128  ...
   <22  > {baz,991628,
        <<232,191,166,252,143,87,251,139,145,217,227,85,5  ...
   <23  > {baz,748772,
        <<210,38,226,254,243,76,211,180,233,33,160,144,25  ...
   <24  > {baz,563034,
        <<52,120,167,54,135,139,205,40,57,250,28,86,34,10  ...
   <25  > {baz,765113,
        <<189,143,122,1,82,185,173,20,161,90,145,16,225,1  ...
   <26  > {baz,510427,
        <<193,192,140,177,72,25,44,199,190,71,144,24,242,  ...
   EOT  (q)uit (p)Digits (k)ill /Regexp -->

For some reason the header of a TABLE.DCL file is getting stuck into
the database in between stopping the node, restarting the node and
starting mnesia again.

	-Vance

On Mon, Oct 18, 2010 at 01:29:30PM -0400, Vance Shipley wrote:
}  On Sat, Jul 03, 2010 at 09:09:31AM +0200, Dan Gudmundsson wrote:
}  }  Are you seeing those in the ets table?
}  
}  Yes I do see the headers in the ets table:
}  
}    4> ets:i(foo).
}    [...]
}    <12  > {log_header,dcl_log,"1.0","4.4.10",'bar@REDACTED',{1287,322033,2 ...
}    [...]
}  
}  }  They should not be in the table, they are what they say the  header of
}  }  a TABLE.DCL file.
}  }  
}  }  Either you or I have some improvements to do :-)
}  
}  I did another database migration today, changing long node name
}  to short, on R13B04 yesterday and experienced the same results.
}  These header records don't show up in the items passed through
}  mnesia:traverse_backup/6 so it doesn't appear to be something
}  I'm doing wrong.
}  
}  I'm going to work on demonstrating the problem as simply as possible.
}  
}  -- 
}  	-Vance
}  
}  }  On Sat, Jul 3, 2010 at 12:28 AM, Vance Shipley <vances@REDACTED> wrote:
}  }  > Is it valid to have records in an mnesia table of this form:
}  }  >
}  }  >  {log_header,dcl_log,"1.0","4.4.10",'foo@REDACTED',{1266,113437,587068}},
}  }  >
}  }  > The tables seem to be working fine even though these entries are
}  }  > there which obviously do not have the correct OID for the table.
}  }  >
}  }  > What I'd like to understand is if I've broken the tables myself
}  }  > or if this type of thing is valid and I should just deal with
}  }  > them when traversing a table with mnesia:foldl/3.
}  }  >
}  }  >        -Vance
}  }  >
}  }  > On Sat, Feb 13, 2010 at 10:19:24PM -0500, Vance Shipley wrote:
}  }  > }  I used the example in the Mnesia User's Guide to backup
}  }  > }  and transform a database from a long node name to a short
}  }  > }  one.  That all seemed to go smoothly enough but now I'm
}  }  > }  noticing these strange records in my tables:
}  }  > }
}  }  > }     {log_header,dcl_log,"1.0","4.4.10",'foo@REDACTED',{1266,113437,587068}},
}  }  > }
}  }  > }  I discovered this using mnesia:foldl/3.  Is this normal
}  }  > }  or have I messed things up?

-- 
	-Vance
-------------- next part --------------
-module(db_transform).

-export([install/4, rename/4, info/1, verify/1]).

-record(foo, {key, value}).
-record(bar, {key, value}).
-record(baz, {key, value}).

-record(acc, {foo = 0, bar = 0, baz = 0}).

install(Nodes, SizeFoo, SizeBar, SizeBaz) ->
	try
		case mnesia:create_schema(Nodes) of
			ok ->
				ok;
			{error, CReason} ->
				throw(CReason)
		end,
		case mnesia:start() of
			ok ->
				ok;
			{error, SReason} ->
				throw(SReason)
		end,
		case mnesia:wait_for_tables([schema], 10000) of
			ok ->
				ok;
			SchemaResult->
				throw(SchemaResult)
		end,
		case mnesia:create_table(foo, [{disc_copies, Nodes},
				{attributes, record_info(fields, foo)}]) of
			{atomic, ok} ->
				ok;
			T1Result ->
				throw(T1Result)
		end,
		case mnesia:create_table(bar, [{disc_copies, Nodes},
				{attributes, record_info(fields, bar)}]) of
			{atomic, ok} ->
				ok;
			T2Result ->
				throw(T2Result)
		end,
		case mnesia:create_table(baz, [{disc_copies, Nodes},
				{attributes, record_info(fields, baz)}]) of
			{atomic, ok} ->
				ok;
			T4Result ->
				throw(T4Result)
		end,
		Tables = [foo, bar, baz],
		case mnesia:wait_for_tables(Tables, 10000) of
			ok ->
				ok;
			TablesResult ->
				throw(TablesResult)
		end,
		crypto:start(),
		Threshold = mnesia:system_info(dump_log_write_threshold) - 1,
		initialize(Tables, Threshold, SizeFoo, SizeBar, SizeBaz),
		{ok, info(Tables)}
	of
		Result ->
			Result
	catch
		throw:Error ->
			mnesia:error_description(Error)
	end.

initialize([], _Threshold, _SizeFoo, _SizeBar, _SizeBaz) ->
	ok;
initialize([foo | Rest], Threshold, SizeFoo, SizeBar, SizeBaz) ->
	batch(foo, SizeFoo, Threshold),
	initialize(Rest, Threshold, SizeFoo, SizeBar, SizeBaz);
initialize([bar | Rest], Threshold, SizeFoo, SizeBar, SizeBaz) ->
	batch(bar, SizeBar, Threshold),
	initialize(Rest, Threshold, SizeFoo, SizeBar, SizeBaz);
initialize([baz | Rest], Threshold, SizeFoo, SizeBar, SizeBaz) ->
	batch(baz, SizeBaz, Threshold),
	initialize(Rest, Threshold, SizeFoo, SizeBar, SizeBaz).
	
batch(_Table, N, _Threshold) when N =< 0 ->
	ok;
batch(Table, N, Threshold) when N =< Threshold ->
	insert(Table, N),
	mnesia:dump_log(),
	ok;
batch(Table, N, Threshold) when N > Threshold ->
	insert(Table, Threshold),
	mnesia:dump_log(),
	batch(Table, N - Threshold, Threshold).

insert(_Table, 0) ->
	ok;
insert(Table, N) ->
	Key = crypto:rand_uniform(0, 1000000),
	Value = crypto:rand_bytes(128),
	Rec = {Table, Key, Value},
	mnesia:dirty_write(Table, Rec),
	insert(Table, N - 1).

rename(From, To, Source, Target) ->
	Fswitch = fun(Node) when Node == From ->
				To;
			(Node) when Node == To ->
				throw({error, already_exists});
			(Other) ->
				Other	
	end,
	Fconvert = fun({schema, db_nodes, Nodes}, Acc) ->
				{[{schema, db_nodes, lists:map(Fswitch,Nodes)}], Acc};
			({schema, Tab, CreateList}, Acc) ->
				Keys = [ram_copies, disc_copies, disc_only_copies, cookie],
				FoptSwitch = fun({cookie, {Now, Node}}) when Node == From ->
							{cookie, {Now, To}};
						({Key, Val}) ->
							case lists:member(Key, Keys) of
								true ->
									{Key, lists:map(Fswitch, Val)};
								false->
									{Key, Val}
							end
				end,
				{[{schema, Tab, lists:map(FoptSwitch, CreateList)}], Acc};
			(#foo{} = Foo, #acc{foo = Count} = Acc) ->
				{[Foo], Acc#acc{foo = Count + 1}};
			(#bar{} = Foo, #acc{bar = Count} = Acc) ->
				{[Foo], Acc#acc{bar = Count + 1}};
			(#baz{} = Foo, #acc{baz = Count} = Acc) ->
				{[Foo], Acc#acc{baz = Count + 1}}
	end,
	mnesia:traverse_backup(Source, Target, Fconvert, #acc{}).

info(Tables) ->
	info(Tables, #acc{}).
info([foo | Rest], Acc) ->
	Size = mnesia:table_info(foo, size),
	info(Rest, Acc#acc{foo = Size});
info([bar | Rest], Acc) ->
	Size = mnesia:table_info(bar, size),
	info(Rest, Acc#acc{bar = Size});
info([baz | Rest], Acc) ->
	Size = mnesia:table_info(baz, size),
	info(Rest, Acc#acc{baz = Size});
info([], Acc) ->
	Acc.

verify(Tables) ->
	verify(Tables, #acc{}).
verify([foo | Rest], Acc) ->
	Fun = fun(#foo{}, Count) -> Count + 1 end,
	Size = count(Fun, foo),
	verify(Rest, Acc#acc{foo = Size});
verify([bar | Rest], Acc) ->
	Fun = fun(#bar{}, Count) -> Count + 1 end,
	Size = count(Fun, bar),
	verify(Rest, Acc#acc{bar = Size});
verify([baz | Rest], Acc) ->
	Fun = fun(#baz{}, Count) -> Count + 1 end,
	Size = count(Fun, baz),
	verify(Rest, Acc#acc{baz = Size});
verify([], Acc) ->
	Acc.

count(Facc, Table) ->
	Ftrans = fun() -> mnesia:foldl(Facc, 0, Table) end,
	case mnesia:transaction(Ftrans) of
		{atomic, Result} ->
			Result;
		{aborted, Reason} ->
			exit(Reason)
	end.



More information about the erlang-questions mailing list