Change node name in mnesia
Vance Shipley
vances@REDACTED
Thu Oct 21 00:41:57 CEST 2010
Dan,
I am able to reproduce the problem with the attached code and
the procedure detailed below. In a nutshell the process outlined
in The Mnesia User's Guide for renaming a database node in a
backup file results in a corrupt database.
http://erlang.org/doc/apps/mnesia/Mnesia_chap7.html#id74479
I used the attached test code, derived from the above example, to
create a sample database, rename the node name from long form to
short and verify it's integrity.
Start a node with a long node name:
$ erl -name long
Eshell V5.8.1 (abort with ^G)
(long@REDACTED)1> rr("db_transform.erl").
[acc,bar,baz,foo]
Build an example data base:
(long@REDACTED)2> db_transform:install([node()], 4000, 500, 25).
{ok,#acc{foo = 3994,bar = 500,baz = 25}}
Back up the example database into the file "long":
(long@REDACTED)3> mnesia:backup(long).
ok
Start a node with a short node name:
$ erl -sname short
Eshell V5.8.1 (abort with ^G)
(short@REDACTED)1> rr("db_transform.erl").
[acc,bar,baz,foo]
Change the node name in the backup file "long" and create backup "short":
(short@REDACTED)2> db_transform:rename('long@REDACTED', 'short@REDACTED', long, short).
{ok,#acc{foo = 3994,bar = 500,baz = 25}}
Create a new database from the backup "short":
(short@REDACTED)3> mnesia:create_schema([node()]).
ok
(short@REDACTED)4> mnesia:start().
ok
(short@REDACTED)5> mnesia:restore(short, [{default_op, recreate_tables}]).
{atomic,[foo,baz,bar]}
Test the database integrity:
(short@REDACTED)6> db_transform:verify([foo, bar, baz]).
#acc{foo = 3994,bar = 500,baz = 25}
The above proves the backup, transform and restore process works.
So far so good. Now below is where the problem can be reproduced.
(short@REDACTED)7> q().
ok
$ erl -sname short
(short@REDACTED)1> mnesia:start().
ok
(short@REDACTED)2> db_transform:verify([foo, bar, baz]).
** exception exit: {function_clause,[{db_transform,'-verify/2-fun-0-',
[{log_header,dcl_log,"1.0","4.4.15",'short@REDACTED',
{1287,613266,224499}},
1899]},
{lists,foldl,3},
{mnesia,do_foldl,8},
{mnesia,foldl,6},
{mnesia_tm,apply_fun,3},
{mnesia_tm,execute_transaction,5},
{db_transform,count,2},
{db_transform,verify,2}]}
in function db_transform:count/2
in call from db_transform:verify/2
Restarting the node is what triggers the problem.
(short@REDACTED)3> ets:i(baz).
<1 > {baz,711590,
<<76,16,157,97,48,225,132,80,36,196,3,199,194,239 ...
<2 > {baz,411561,
<<173,175,232,135,28,118,233,210,111,49,169,19,88 ...
<3 > {baz,909239,
<<135,214,14,32,8,121,144,64,196,35,186,167,63,24 ...
<4 > {baz,463433,
<<66,156,188,32,49,157,54,160,133,241,196,78,22,2 ...
<5 > {baz,997452,
<<248,253,234,156,76,130,48,212,89,197,186,210,21 ...
<6 > {baz,797390,
<<25,1,27,228,30,234,2,147,80,80,167,58,230,54,11 ...
<7 > {baz,879889,
<<249,45,35,209,179,173,217,158,208,227,220,67,20 ...
<8 > {baz,604804,
<<120,75,101,145,118,116,39,218,204,246,198,162,3 ...
<9 > {baz,807967,
<<58,209,110,188,212,128,136,15,133,56,166,13,30, ...
<10 > {baz,388388,
<<143,53,248,42,115,146,218,222,246,67,10,173,185 ...
<11 > {baz,890638,
<<11,137,19,231,155,97,254,94,79,68,94,82,156,190 ...
<12 > {baz,865475,
<<169,173,174,222,235,89,94,90,11,192,50,72,37,43 ...
<13 > {baz,595736,
<<43,98,245,177,0,79,54,225,36,201,24,188,195,240 ...
<14 > {log_header,dcl_log,"1.0","4.4.15",'short@REDACTED',
...
<15 > {baz,771466,
<<39,59,205,76,71,40,80,85,65,229,224,198,109,132 ...
<16 > {baz,834821,
<<76,65,143,50,217,159,155,39,129,56,67,59,73,95, ...
<17 > {baz,534651,
<<72,192,141,91,154,171,201,126,135,7,213,143,13, ...
<18 > {baz,551117,
<<85,185,77,144,114,113,203,93,206,29,122,106,52, ...
<19 > {baz,224818,
<<120,27,23,224,251,51,254,69,215,104,11,93,135,5 ...
<20 > {baz,358442,
<<255,165,10,214,6,121,98,31,231,82,52,194,27,94, ...
<21 > {baz,382522,
<<188,190,108,27,51,81,186,175,215,95,168,203,128 ...
<22 > {baz,991628,
<<232,191,166,252,143,87,251,139,145,217,227,85,5 ...
<23 > {baz,748772,
<<210,38,226,254,243,76,211,180,233,33,160,144,25 ...
<24 > {baz,563034,
<<52,120,167,54,135,139,205,40,57,250,28,86,34,10 ...
<25 > {baz,765113,
<<189,143,122,1,82,185,173,20,161,90,145,16,225,1 ...
<26 > {baz,510427,
<<193,192,140,177,72,25,44,199,190,71,144,24,242, ...
EOT (q)uit (p)Digits (k)ill /Regexp -->
For some reason the header of a TABLE.DCL file is getting stuck into
the database in between stopping the node, restarting the node and
starting mnesia again.
-Vance
On Mon, Oct 18, 2010 at 01:29:30PM -0400, Vance Shipley wrote:
} On Sat, Jul 03, 2010 at 09:09:31AM +0200, Dan Gudmundsson wrote:
} } Are you seeing those in the ets table?
}
} Yes I do see the headers in the ets table:
}
} 4> ets:i(foo).
} [...]
} <12 > {log_header,dcl_log,"1.0","4.4.10",'bar@REDACTED',{1287,322033,2 ...
} [...]
}
} } They should not be in the table, they are what they say the header of
} } a TABLE.DCL file.
} }
} } Either you or I have some improvements to do :-)
}
} I did another database migration today, changing long node name
} to short, on R13B04 yesterday and experienced the same results.
} These header records don't show up in the items passed through
} mnesia:traverse_backup/6 so it doesn't appear to be something
} I'm doing wrong.
}
} I'm going to work on demonstrating the problem as simply as possible.
}
} --
} -Vance
}
} } On Sat, Jul 3, 2010 at 12:28 AM, Vance Shipley <vances@REDACTED> wrote:
} } > Is it valid to have records in an mnesia table of this form:
} } >
} } > {log_header,dcl_log,"1.0","4.4.10",'foo@REDACTED',{1266,113437,587068}},
} } >
} } > The tables seem to be working fine even though these entries are
} } > there which obviously do not have the correct OID for the table.
} } >
} } > What I'd like to understand is if I've broken the tables myself
} } > or if this type of thing is valid and I should just deal with
} } > them when traversing a table with mnesia:foldl/3.
} } >
} } > -Vance
} } >
} } > On Sat, Feb 13, 2010 at 10:19:24PM -0500, Vance Shipley wrote:
} } > } I used the example in the Mnesia User's Guide to backup
} } > } and transform a database from a long node name to a short
} } > } one. That all seemed to go smoothly enough but now I'm
} } > } noticing these strange records in my tables:
} } > }
} } > } {log_header,dcl_log,"1.0","4.4.10",'foo@REDACTED',{1266,113437,587068}},
} } > }
} } > } I discovered this using mnesia:foldl/3. Is this normal
} } > } or have I messed things up?
--
-Vance
-------------- next part --------------
-module(db_transform).
-export([install/4, rename/4, info/1, verify/1]).
-record(foo, {key, value}).
-record(bar, {key, value}).
-record(baz, {key, value}).
-record(acc, {foo = 0, bar = 0, baz = 0}).
install(Nodes, SizeFoo, SizeBar, SizeBaz) ->
try
case mnesia:create_schema(Nodes) of
ok ->
ok;
{error, CReason} ->
throw(CReason)
end,
case mnesia:start() of
ok ->
ok;
{error, SReason} ->
throw(SReason)
end,
case mnesia:wait_for_tables([schema], 10000) of
ok ->
ok;
SchemaResult->
throw(SchemaResult)
end,
case mnesia:create_table(foo, [{disc_copies, Nodes},
{attributes, record_info(fields, foo)}]) of
{atomic, ok} ->
ok;
T1Result ->
throw(T1Result)
end,
case mnesia:create_table(bar, [{disc_copies, Nodes},
{attributes, record_info(fields, bar)}]) of
{atomic, ok} ->
ok;
T2Result ->
throw(T2Result)
end,
case mnesia:create_table(baz, [{disc_copies, Nodes},
{attributes, record_info(fields, baz)}]) of
{atomic, ok} ->
ok;
T4Result ->
throw(T4Result)
end,
Tables = [foo, bar, baz],
case mnesia:wait_for_tables(Tables, 10000) of
ok ->
ok;
TablesResult ->
throw(TablesResult)
end,
crypto:start(),
Threshold = mnesia:system_info(dump_log_write_threshold) - 1,
initialize(Tables, Threshold, SizeFoo, SizeBar, SizeBaz),
{ok, info(Tables)}
of
Result ->
Result
catch
throw:Error ->
mnesia:error_description(Error)
end.
initialize([], _Threshold, _SizeFoo, _SizeBar, _SizeBaz) ->
ok;
initialize([foo | Rest], Threshold, SizeFoo, SizeBar, SizeBaz) ->
batch(foo, SizeFoo, Threshold),
initialize(Rest, Threshold, SizeFoo, SizeBar, SizeBaz);
initialize([bar | Rest], Threshold, SizeFoo, SizeBar, SizeBaz) ->
batch(bar, SizeBar, Threshold),
initialize(Rest, Threshold, SizeFoo, SizeBar, SizeBaz);
initialize([baz | Rest], Threshold, SizeFoo, SizeBar, SizeBaz) ->
batch(baz, SizeBaz, Threshold),
initialize(Rest, Threshold, SizeFoo, SizeBar, SizeBaz).
batch(_Table, N, _Threshold) when N =< 0 ->
ok;
batch(Table, N, Threshold) when N =< Threshold ->
insert(Table, N),
mnesia:dump_log(),
ok;
batch(Table, N, Threshold) when N > Threshold ->
insert(Table, Threshold),
mnesia:dump_log(),
batch(Table, N - Threshold, Threshold).
insert(_Table, 0) ->
ok;
insert(Table, N) ->
Key = crypto:rand_uniform(0, 1000000),
Value = crypto:rand_bytes(128),
Rec = {Table, Key, Value},
mnesia:dirty_write(Table, Rec),
insert(Table, N - 1).
rename(From, To, Source, Target) ->
Fswitch = fun(Node) when Node == From ->
To;
(Node) when Node == To ->
throw({error, already_exists});
(Other) ->
Other
end,
Fconvert = fun({schema, db_nodes, Nodes}, Acc) ->
{[{schema, db_nodes, lists:map(Fswitch,Nodes)}], Acc};
({schema, Tab, CreateList}, Acc) ->
Keys = [ram_copies, disc_copies, disc_only_copies, cookie],
FoptSwitch = fun({cookie, {Now, Node}}) when Node == From ->
{cookie, {Now, To}};
({Key, Val}) ->
case lists:member(Key, Keys) of
true ->
{Key, lists:map(Fswitch, Val)};
false->
{Key, Val}
end
end,
{[{schema, Tab, lists:map(FoptSwitch, CreateList)}], Acc};
(#foo{} = Foo, #acc{foo = Count} = Acc) ->
{[Foo], Acc#acc{foo = Count + 1}};
(#bar{} = Foo, #acc{bar = Count} = Acc) ->
{[Foo], Acc#acc{bar = Count + 1}};
(#baz{} = Foo, #acc{baz = Count} = Acc) ->
{[Foo], Acc#acc{baz = Count + 1}}
end,
mnesia:traverse_backup(Source, Target, Fconvert, #acc{}).
info(Tables) ->
info(Tables, #acc{}).
info([foo | Rest], Acc) ->
Size = mnesia:table_info(foo, size),
info(Rest, Acc#acc{foo = Size});
info([bar | Rest], Acc) ->
Size = mnesia:table_info(bar, size),
info(Rest, Acc#acc{bar = Size});
info([baz | Rest], Acc) ->
Size = mnesia:table_info(baz, size),
info(Rest, Acc#acc{baz = Size});
info([], Acc) ->
Acc.
verify(Tables) ->
verify(Tables, #acc{}).
verify([foo | Rest], Acc) ->
Fun = fun(#foo{}, Count) -> Count + 1 end,
Size = count(Fun, foo),
verify(Rest, Acc#acc{foo = Size});
verify([bar | Rest], Acc) ->
Fun = fun(#bar{}, Count) -> Count + 1 end,
Size = count(Fun, bar),
verify(Rest, Acc#acc{bar = Size});
verify([baz | Rest], Acc) ->
Fun = fun(#baz{}, Count) -> Count + 1 end,
Size = count(Fun, baz),
verify(Rest, Acc#acc{baz = Size});
verify([], Acc) ->
Acc.
count(Facc, Table) ->
Ftrans = fun() -> mnesia:foldl(Facc, 0, Table) end,
case mnesia:transaction(Ftrans) of
{atomic, Result} ->
Result;
{aborted, Reason} ->
exit(Reason)
end.
More information about the erlang-bugs
mailing list