Mnesia patch, was: Re: Mnesia and partitioning big tables: Doc, it hurts when...

[Scott Lystig Fritchie]

Following up on my earlier message...

... I'd mentioned that I'd had a Mnesia 'disc_copies' table with 800K
entries in it.  size(term_to_binary(Record)) = 308, on average.  The
inital attempt to add a fragment via:

    mnesia:change_table_frag('Profile', {add_frag, [node()]})

... well, I aborted it after over an hour.  The VM was unusable: each
line of output by mnesia:info() took 15-20 seconds to appear.

Using the patch below, the first mnesia:change_table_frag/2 call takes
24 seconds.  Adding the 20th fragment takes 1.6 seconds.

The patch is relative to the R10B-9 release.  It patches R11B-1 with
only a few "offset X lines" warnings, but I've only tested it with
R10B-9.

It would be really, really nice if:

   * Another set of eyes can find any bugs?
      * There's a possible table handle leak, I guess, if you're
      only 1 table away from the limit on # of ETS tables....
   * If no bugs, inclusion into the Mnesia source distribution

-Scott

--- snip --- snip --- snip --- snip --- snip --- snip ---

diff -ur ./mnesia.hrl /home/fritchie/src/mnesia-exp/src/mnesia.hrl
--- ./mnesia.hrl	2006-01-09 13:48:16.000000000 -0600
+++ /home/fritchie/src/mnesia-exp/src/mnesia.hrl	2006-09-02 00:06:59.004182832 -0500
@@ -20,7 +20,7 @@
 
 -define(ets_lookup(Tab, Key), ets:lookup(Tab, Key)).
 -define(ets_lookup_element(Tab, Key, Pos), ets:lookup_element(Tab, Key, Pos)).
--define(ets_insert(Tab, Rec), ets:insert(Tab, Rec)).
+-define(ets_insert(Tab, Rec), mnesia_lib:redirect_ets_insert(Rec, Tab)).
 -define(ets_delete(Tab, Key), ets:delete(Tab, Key)).
 -define(ets_match_delete(Tab, Pat), ets:match_delete(Tab, Pat)).
 -define(ets_match_object(Tab, Pat), ets:match_object(Tab, Pat)).
@@ -33,8 +33,9 @@
 -define(ets_prev(Tab, Key), ets:prev(Tab, Key)).
 -define(ets_slot(Tab, Pos), ets:slot(Tab, Pos)).
 -define(ets_new_table(Tab, Props), ets:new(Tab, Props)).
--define(ets_delete_table(Tab), ets:delete(Tab)).
+-define(ets_delete_table(Tab), mnesia_lib:redirect_ets_delete(Tab)).
 -define(ets_fixtable(Tab, Bool), ets:fixtable(Tab, Bool)).
+-define(ets_tab2list(Tab), ets:tab2list(Tab)).
 
 -define(catch_val(Var), (catch ?ets_lookup_element(mnesia_gvar, Var, 2))).
 
diff -ur ./mnesia_lib.erl /home/fritchie/src/mnesia-exp/src/mnesia_lib.erl
--- ./mnesia_lib.erl	2006-01-09 13:48:16.000000000 -0600
+++ /home/fritchie/src/mnesia-exp/src/mnesia_lib.erl	2006-09-02 00:19:53.790397424 -0500
@@ -151,6 +151,9 @@
 	 view/2,
 	 warning/2,
 
+	 redirect_ets_insert/2,
+	 redirect_ets_delete/1,
+
 	 is_debug_compiled/0,
 	 activate_debug_fun/5,
 	 deactivate_debug_fun/3,
@@ -1304,4 +1307,25 @@
     is_debug_compiled() -> false.
 -endif.   
 
-
+redirect_ets_insert(Rec, Tab) ->
+    TabName = ets:info(Tab, name),
+    if TabName == mnesia_trans_store, is_tuple(Rec), element(1, Rec) == op ->
+	    [{etab_set, Etab_Set}] = ets:lookup(Tab, etab_set),
+	    %% TODO: Is it worth a test if {op, look_elsewhere} is already
+	    %%       in Tab and insert only if it doesn't exist?
+	    ets:insert(Tab, {op, look_elsewhere}),
+	    %% Need a unique key, make_ref() is perfect.
+	    ets:insert(Etab_Set, {make_ref(), Rec});
+       true ->
+	    ets:insert(Tab, Rec)
+    end.
+
+redirect_ets_delete(Tab) ->
+    TabName = ets:info(Tab, name),
+    if TabName == mnesia_trans_store ->
+	    [{etab_set, Etab_Set}] = ?ets_lookup(Tab, etab_set),
+	    ets:delete(Etab_Set);
+       true ->
+	    ok
+    end,
+    ets:delete(Tab).
diff -ur ./mnesia_tm.erl /home/fritchie/src/mnesia-exp/src/mnesia_tm.erl
--- ./mnesia_tm.erl	2006-01-09 13:48:16.000000000 -0600
+++ /home/fritchie/src/mnesia-exp/src/mnesia_tm.erl	2006-09-02 00:19:43.331987344 -0500
@@ -219,15 +219,17 @@
 	    end;
 	
 	{From, start_outer} -> %% Create and associate ets_tab with Tid
-	    case catch ?ets_new_table(mnesia_trans_store, [bag, public]) of
+	    case catch ({?ets_new_table(mnesia_trans_store, [bag, public]),
+			 ?ets_new_table(mnesia_trans_store_Set, [ordered_set, public])}) of
 		{'EXIT', Reason} -> %% system limit
 		    Msg = "Cannot create an ets table for the "
 	                  "local transaction store",
 		    reply(From, {error, {system_limit, Msg, Reason}}, State);
-		Etab ->
+		{Etab, Etab_Set} ->
 		    tmlink(From),
 		    C = mnesia_recover:incr_trans_tid_serial(),
 		    ?ets_insert(Etab, {nodes, node()}),
+		    ?ets_insert(Etab, {etab_set, Etab_Set}),
 		    Tid = #tid{pid = tmpid(From), counter = C},
 		    A2 = [{Tid , [Etab]} | Coordinators],
 		    S2 = State#state{coordinators = A2},
@@ -334,12 +336,14 @@
 	    end;
 	
 	{From, {add_store, Tid}} -> %% new store for nested  transaction
-	    case catch ?ets_new_table(mnesia_trans_store, [bag, public]) of
+	    case catch ({?ets_new_table(mnesia_trans_store, [bag, public]),
+			 ?ets_new_table(mnesia_trans_store_Set, [ordered_set, public])}) of
 		{'EXIT', Reason} -> %% system limit
 		    Msg = "Cannot create an ets table for a nested "
                           "local transaction store",
 		    reply(From, {error, {system_limit, Msg, Reason}}, State);
-		Etab ->
+		{Etab, Etab_Set} ->
+		    ?ets_insert(Etab, {etab_set, Etab_Set}),
 		    A2 = add_coord_store(Coordinators, Tid, Etab),
 		    reply(From, {new_store, Etab},
 			  State#state{coordinators = A2})
@@ -1187,7 +1191,10 @@
     P2 = prepare_items(Tid, Tab, Key, Items, Prep),
     do_arrange(Tid, Store, ?ets_next(Store, Oid), P2, N + 1);
 do_arrange(Tid, Store, SchemaKey, Prep, N) when SchemaKey == op ->
-    Items = ?ets_lookup(Store, SchemaKey), %% Store is a bag
+    %% Get the list of 'op' tuples that used to be stored in the Store bag,
+    %% then strip off each 2-tuple wrapper.
+    [{etab_set, Etab_Set}] = ?ets_lookup(Store, etab_set),
+    Items = [Val || {_Ref, Val} <- ?ets_tab2list(Etab_Set)],
     P2 = prepare_schema_items(Tid, Items, Prep),
     do_arrange(Tid, Store, ?ets_next(Store, SchemaKey), P2, N + 1);
 do_arrange(Tid, Store, RestoreKey, Prep, N) when RestoreKey == restore_op ->