X-Git-Url: https://git.sur5r.net/?a=blobdiff_plain;f=servers%2Fslapd%2Fback-mdb%2Ftools.c;h=4a53962ecc883f7bbd38abf9c657c00bb3390a99;hb=9d5d7eba7192c6dab7c6f8ef6fb3d083b53565db;hp=79ef877f43334f0b51b847e82a1931996533cc51;hpb=eba99f5e862e0792ebd05e8c5f9cd9175a8ca8f1;p=openldap

diff --git a/servers/slapd/back-mdb/tools.c b/servers/slapd/back-mdb/tools.c
index 79ef877f43..4a53962ecc 100644
--- a/servers/slapd/back-mdb/tools.c
+++ b/servers/slapd/back-mdb/tools.c
@@ -2,7 +2,7 @@
 /* $OpenLDAP$ */
 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
  *
- * Copyright 2011 The OpenLDAP Foundation.
+ * Copyright 2011-2012 The OpenLDAP Foundation.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -24,6 +24,7 @@
 #include "back-mdb.h"
 #include "idl.h"
 
+#ifdef MDB_TOOL_IDL_CACHING
 static int mdb_tool_idl_flush( BackendDB *be, MDB_txn *txn );
 
 #define	IDBLOCK	1024
@@ -38,13 +39,20 @@ typedef struct mdb_tool_idl_cache {
 	mdb_tool_idl_cache_entry *head, *tail;
 	ID first, last;
 	int count;
+	short offset;
+	short flags;
 } mdb_tool_idl_cache;
+#define WAS_FOUND	0x01
+#define WAS_RANGE	0x02
 
-static mdb_tool_idl_cache_entry *mdb_tool_idl_free_list;
-static Avlnode *mdb_tool_roots[MDB_INDICES];
+#define MDB_TOOL_IDL_FLUSH(be, txn)	mdb_tool_idl_flush(be, txn)
+#else
+#define MDB_TOOL_IDL_FLUSH(be, txn)
+#endif /* MDB_TOOL_IDL_CACHING */
 
 static MDB_txn *txn = NULL, *txi = NULL;
 static MDB_cursor *cursor = NULL, *idcursor = NULL;
+static MDB_cursor *mcp = NULL, *mcd = NULL;
 static MDB_val key, data;
 static ID previd = NOID;
 
@@ -63,17 +71,16 @@ static int		tool_scope;
 static Filter		*tool_filter;
 static Entry		*tool_next_entry;
 
-#if 0
 static ID mdb_tool_ix_id;
 static Operation *mdb_tool_ix_op;
-static int *mdb_tool_index_threads, mdb_tool_index_tcount;
-static void *mdb_tool_index_rec;
+static MDB_txn *mdb_tool_ix_txn;
+static int mdb_tool_index_tcount, mdb_tool_threads;
+static IndexRec *mdb_tool_index_rec;
 static struct mdb_info *mdb_tool_info;
 static ldap_pvt_thread_mutex_t mdb_tool_index_mutex;
 static ldap_pvt_thread_cond_t mdb_tool_index_cond_main;
 static ldap_pvt_thread_cond_t mdb_tool_index_cond_work;
 static void * mdb_tool_index_task( void *ctx, void *ptr );
-#endif
 
 static int	mdb_writes, mdb_writes_per_commit;
 
@@ -90,29 +97,30 @@ int mdb_tool_entry_open(
 	else
 		mdb_writes_per_commit = 1;
 
-#if 0
 	/* Set up for threaded slapindex */
 	if (( slapMode & (SLAP_TOOL_QUICK|SLAP_TOOL_READONLY)) == SLAP_TOOL_QUICK ) {
 		if ( !mdb_tool_info ) {
+			struct mdb_info *mdb = (struct mdb_info *) be->be_private;
 			ldap_pvt_thread_mutex_init( &mdb_tool_index_mutex );
 			ldap_pvt_thread_cond_init( &mdb_tool_index_cond_main );
 			ldap_pvt_thread_cond_init( &mdb_tool_index_cond_work );
-			if ( mdb->bi_nattrs ) {
+			if ( mdb->mi_nattrs ) {
 				int i;
-				mdb_tool_index_threads = ch_malloc( slap_tool_thread_max * sizeof( int ));
-				mdb_tool_index_rec = ch_malloc( mdb->bi_nattrs * sizeof( IndexRec ));
-				mdb_tool_index_tcount = slap_tool_thread_max - 1;
-				for (i=1; i<slap_tool_thread_max; i++) {
-					int *ptr = ch_malloc( sizeof( int ));
-					*ptr = i;
-					ldap_pvt_thread_pool_submit( &connection_pool,
-						mdb_tool_index_task, ptr );
+				mdb_tool_threads = slap_tool_thread_max - 1;
+				if ( mdb_tool_threads > 1 ) {
+					mdb_tool_index_rec = ch_calloc( mdb->mi_nattrs, sizeof( IndexRec ));
+					mdb_tool_index_tcount = mdb_tool_threads - 1;
+					for (i=1; i<mdb_tool_threads; i++) {
+						int *ptr = ch_malloc( sizeof( int ));
+						*ptr = i;
+						ldap_pvt_thread_pool_submit( &connection_pool,
+							mdb_tool_index_task, ptr );
+					}
+					mdb_tool_info = mdb;
 				}
 			}
-			mdb_tool_info = mdb;
 		}
 	}
-#endif
 
 	return 0;
 }
@@ -120,22 +128,21 @@ int mdb_tool_entry_open(
 int mdb_tool_entry_close(
 	BackendDB *be )
 {
-#if 0
 	if ( mdb_tool_info ) {
 		slapd_shutdown = 1;
 		ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex );
 
 		/* There might still be some threads starting */
-		while ( mdb_tool_index_tcount ) {
+		while ( mdb_tool_index_tcount > 0 ) {
 			ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main,
 					&mdb_tool_index_mutex );
 		}
 
-		mdb_tool_index_tcount = slap_tool_thread_max - 1;
+		mdb_tool_index_tcount = mdb_tool_threads - 1;
 		ldap_pvt_thread_cond_broadcast( &mdb_tool_index_cond_work );
 
 		/* Make sure all threads are stopped */
-		while ( mdb_tool_index_tcount ) {
+		while ( mdb_tool_index_tcount > 0 ) {
 			ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main,
 				&mdb_tool_index_mutex );
 		}
@@ -143,11 +150,9 @@ int mdb_tool_entry_close(
 
 		mdb_tool_info = NULL;
 		slapd_shutdown = 0;
-		ch_free( mdb_tool_index_threads );
 		ch_free( mdb_tool_index_rec );
-		mdb_tool_index_tcount = slap_tool_thread_max - 1;
+		mdb_tool_index_tcount = mdb_tool_threads - 1;
 	}
-#endif
 
 	if( idcursor ) {
 		mdb_cursor_close( idcursor );
@@ -157,8 +162,8 @@ int mdb_tool_entry_close(
 		mdb_cursor_close( cursor );
 		cursor = NULL;
 	}
-	mdb_tool_idl_flush( be, txn );
 	if( txn ) {
+		MDB_TOOL_IDL_FLUSH( be, txn );
 		if ( mdb_txn_commit( txn ))
 			return -1;
 		txn = NULL;
@@ -286,7 +291,7 @@ ID mdb_tool_dn2id_get(
 	op.o_tmpmemctx = NULL;
 	op.o_tmpmfuncs = &ch_mfuncs;
 
-	rc = mdb_dn2id( &op, txn, dn, &id, NULL, NULL );
+	rc = mdb_dn2id( &op, txn, NULL, dn, &id, NULL, NULL );
 	if ( rc == MDB_NOTFOUND )
 		return NOID;
 
@@ -330,8 +335,10 @@ mdb_tool_entry_get_int( BackendDB *be, ID id, Entry **ep )
 		rc = mdb_id2name( &op, txn, &idcursor, id, &dn, &ndn );
 		if ( rc  ) {
 			rc = LDAP_OTHER;
-			mdb_entry_return( &op, e );
-			e = NULL;
+			if ( e ) {
+				mdb_entry_return( &op, e );
+				e = NULL;
+			}
 			goto done;
 		}
 		if ( tool_base != NULL ) {
@@ -364,7 +371,24 @@ Entry*
 mdb_tool_entry_get( BackendDB *be, ID id )
 {
 	Entry *e = NULL;
+	int rc;
 
+	if ( !txn ) {
+		struct mdb_info *mdb = (struct mdb_info *) be->be_private;
+		rc = mdb_txn_begin( mdb->mi_dbenv, NULL,
+			(slapMode & SLAP_TOOL_READONLY) ? MDB_RDONLY : 0, &txn );
+		if ( rc )
+			return NULL;
+	}
+	if ( !cursor ) {
+		struct mdb_info *mdb = (struct mdb_info *) be->be_private;
+		rc = mdb_cursor_open( txn, mdb->mi_id2entry, &cursor );
+		if ( rc ) {
+			mdb_txn_abort( txn );
+			txn = NULL;
+			return NULL;
+		}
+	}
 	(void)mdb_tool_entry_get_int( be, id, &e );
 	return e;
 }
@@ -388,7 +412,7 @@ static int mdb_tool_next_id(
 		return 0;
 	}
 
-	rc = mdb_dn2id( op, tid, &ndn, &id, NULL, &nmatched );
+	rc = mdb_dn2id( op, tid, mcp, &ndn, &id, NULL, &nmatched );
 	if ( rc == MDB_NOTFOUND ) {
 		if ( !be_issuffix( op->o_bd, &ndn ) ) {
 			ID eid = e->e_id;
@@ -414,7 +438,7 @@ static int mdb_tool_next_id(
 				pid = id;
 			}
 		}
-		rc = mdb_next_id( op->o_bd, tid, &e->e_id );
+		rc = mdb_next_id( op->o_bd, idcursor, &e->e_id );
 		if ( rc ) {
 			snprintf( text->bv_val, text->bv_len,
 				"next_id failed: %s (%d)",
@@ -423,7 +447,7 @@ static int mdb_tool_next_id(
 			"=> mdb_tool_next_id: %s\n", text->bv_val, 0, 0 );
 			return rc;
 		}
-		rc = mdb_dn2id_add( op, tid, pid, e );
+		rc = mdb_dn2id_add( op, mcp, mcd, pid, e );
 		if ( rc ) {
 			snprintf( text->bv_val, text->bv_len,
 				"dn2id_add failed: %s (%d)",
@@ -447,7 +471,7 @@ static int mdb_tool_next_id(
 			key.mv_data = &e->e_id;
 			data.mv_size = 0;
 			data.mv_data = NULL;
-			rc = mdb_put( tid, mdb->mi_id2entry, &key, &data, MDB_NOOVERWRITE );
+			rc = mdb_cursor_put( idcursor, &key, &data, MDB_NOOVERWRITE );
 			if ( rc == MDB_KEYEXIST )
 				rc = 0;
 			if ( rc ) {
@@ -489,14 +513,14 @@ mdb_tool_index_add(
 	if ( !mdb->mi_nattrs )
 		return 0;
 
-#if 0
-	if ( slapMode & SLAP_TOOL_QUICK ) {
+	if ( mdb_tool_threads > 1 ) {
 		IndexRec *ir;
 		int i, rc;
 		Attribute *a;
 
 		ir = mdb_tool_index_rec;
-		memset(ir, 0, mdb->bi_nattrs * sizeof( IndexRec ));
+		for (i=0; i<mdb->mi_nattrs; i++)
+			ir[i].ir_attrs = NULL;
 
 		for ( a = e->e_attrs; a != NULL; a = a->a_next ) {
 			rc = mdb_index_recset( mdb, a, a->a_desc->ad_type,
@@ -504,39 +528,49 @@ mdb_tool_index_add(
 			if ( rc )
 				return rc;
 		}
+		for (i=0; i<mdb->mi_nattrs; i++) {
+			if ( !ir[i].ir_ai )
+				break;
+			rc = mdb_cursor_open( txn, ir[i].ir_ai->ai_dbi,
+				 &ir[i].ir_ai->ai_cursor );
+			if ( rc )
+				return rc;
+		}
 		mdb_tool_ix_id = e->e_id;
 		mdb_tool_ix_op = op;
+		mdb_tool_ix_txn = txn;
 		ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex );
 		/* Wait for all threads to be ready */
 		while ( mdb_tool_index_tcount ) {
 			ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main,
 				&mdb_tool_index_mutex );
 		}
-		for ( i=1; i<slap_tool_thread_max; i++ )
-			mdb_tool_index_threads[i] = LDAP_BUSY;
-		mdb_tool_index_tcount = slap_tool_thread_max - 1;
-		ldap_pvt_thread_cond_broadcast( &mdb_tool_index_cond_work );
+
+		for ( i=1; i<mdb_tool_threads; i++ )
+			mdb_tool_index_rec[i].ir_i = LDAP_BUSY;
+		mdb_tool_index_tcount = mdb_tool_threads - 1;
 		ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex );
-		rc = mdb_index_recrun( op, mdb, ir, e->e_id, 0 );
+		ldap_pvt_thread_cond_broadcast( &mdb_tool_index_cond_work );
+
+		rc = mdb_index_recrun( op, txn, mdb, ir, e->e_id, 0 );
 		if ( rc )
 			return rc;
 		ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex );
-		for ( i=1; i<slap_tool_thread_max; i++ ) {
-			if ( mdb_tool_index_threads[i] == LDAP_BUSY ) {
+		for ( i=1; i<mdb_tool_threads; i++ ) {
+			if ( mdb_tool_index_rec[i].ir_i == LDAP_BUSY ) {
 				ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main,
 					&mdb_tool_index_mutex );
 				i--;
 				continue;
 			}
-			if ( mdb_tool_index_threads[i] ) {
-				rc = mdb_tool_index_threads[i];
+			if ( mdb_tool_index_rec[i].ir_i ) {
+				rc = mdb_tool_index_rec[i].ir_i;
 				break;
 			}
 		}
 		ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex );
 		return rc;
 	} else
-#endif
 	{
 		return mdb_index_entry_add( op, txn, e );
 	}
@@ -565,16 +599,46 @@ ID mdb_tool_entry_put(
 	mdb = (struct mdb_info *) be->be_private;
 
 	if ( !txn ) {
-	rc = mdb_txn_begin( mdb->mi_dbenv, NULL, 0, &txn );
-	if( rc != 0 ) {
-		snprintf( text->bv_val, text->bv_len,
-			"txn_begin failed: %s (%d)",
-			mdb_strerror(rc), rc );
-		Debug( LDAP_DEBUG_ANY,
-			"=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
-			 text->bv_val, 0, 0 );
-		return NOID;
-	}
+		rc = mdb_txn_begin( mdb->mi_dbenv, NULL, 0, &txn );
+		if( rc != 0 ) {
+			snprintf( text->bv_val, text->bv_len,
+				"txn_begin failed: %s (%d)",
+				mdb_strerror(rc), rc );
+			Debug( LDAP_DEBUG_ANY,
+				"=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
+				 text->bv_val, 0, 0 );
+			return NOID;
+		}
+		rc = mdb_cursor_open( txn, mdb->mi_id2entry, &idcursor );
+		if( rc != 0 ) {
+			snprintf( text->bv_val, text->bv_len,
+				"cursor_open failed: %s (%d)",
+				mdb_strerror(rc), rc );
+			Debug( LDAP_DEBUG_ANY,
+				"=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
+				 text->bv_val, 0, 0 );
+			return NOID;
+		}
+		rc = mdb_cursor_open( txn, mdb->mi_dn2id, &mcp );
+		if( rc != 0 ) {
+			snprintf( text->bv_val, text->bv_len,
+				"cursor_open failed: %s (%d)",
+				mdb_strerror(rc), rc );
+			Debug( LDAP_DEBUG_ANY,
+				"=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
+				 text->bv_val, 0, 0 );
+			return NOID;
+		}
+		rc = mdb_cursor_open( txn, mdb->mi_dn2id, &mcd );
+		if( rc != 0 ) {
+			snprintf( text->bv_val, text->bv_len,
+				"cursor_open failed: %s (%d)",
+				mdb_strerror(rc), rc );
+			Debug( LDAP_DEBUG_ANY,
+				"=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
+				 text->bv_val, 0, 0 );
+			return NOID;
+		}
 	}
 
 	op.o_hdr = &ohdr;
@@ -600,7 +664,7 @@ ID mdb_tool_entry_put(
 
 
 	/* id2entry index */
-	rc = mdb_id2entry_add( &op, txn, e );
+	rc = mdb_id2entry_add( &op, txn, idcursor, e );
 	if( rc != 0 ) {
 		snprintf( text->bv_val, text->bv_len,
 				"id2entry_add failed: err=%d", rc );
@@ -614,10 +678,14 @@ done:
 	if( rc == 0 ) {
 		mdb_writes++;
 		if ( mdb_writes >= mdb_writes_per_commit ) {
-			mdb_tool_idl_flush( be, txn );
+			unsigned i;
+			MDB_TOOL_IDL_FLUSH( be, txn );
 			rc = mdb_txn_commit( txn );
+			for ( i=0; i<mdb->mi_nattrs; i++ )
+				mdb->mi_attrs[i]->ai_cursor = NULL;
 			mdb_writes = 0;
 			txn = NULL;
+			idcursor = NULL;
 			if( rc != 0 ) {
 				snprintf( text->bv_val, text->bv_len,
 						"txn_commit failed: %s (%d)",
@@ -630,8 +698,13 @@ done:
 		}
 
 	} else {
+		unsigned i;
 		mdb_txn_abort( txn );
 		txn = NULL;
+		idcursor = NULL;
+		for ( i=0; i<mdb->mi_nattrs; i++ )
+			mdb->mi_attrs[i]->ai_cursor = NULL;
+		mdb_writes = 0;
 		snprintf( text->bv_val, text->bv_len,
 			"txn_aborted! %s (%d)",
 			rc == LDAP_OTHER ? "Internal error" :
@@ -710,22 +783,6 @@ int mdb_tool_entry_reindex(
 		mi->mi_nattrs = i;
 	}
 
-	if ( slapMode & SLAP_TRUNCATE_MODE ) {
-		int i;
-		for ( i=0; i < mi->mi_nattrs; i++ ) {
-			rc = mdb_drop( txn, mi->mi_attrs[i]->ai_dbi, 0 );
-			if ( rc ) {
-				Debug( LDAP_DEBUG_ANY,
-					LDAP_XSTRING(mdb_tool_entry_reindex)
-					": (Truncate) mdb_drop(%s) failed: %s (%d)\n",
-					mi->mi_attrs[i]->ai_desc->ad_type->sat_cname.bv_val,
-					mdb_strerror(rc), rc );
-				return -1;
-			}
-		}
-		slapMode ^= SLAP_TRUNCATE_MODE;
-	}
-
 	e = mdb_tool_entry_get( be, id );
 
 	if( e == NULL ) {
@@ -747,16 +804,30 @@ int mdb_tool_entry_reindex(
 		}
 	}
 
+	if ( slapMode & SLAP_TRUNCATE_MODE ) {
+		int i;
+		for ( i=0; i < mi->mi_nattrs; i++ ) {
+			rc = mdb_drop( txi, mi->mi_attrs[i]->ai_dbi, 0 );
+			if ( rc ) {
+				Debug( LDAP_DEBUG_ANY,
+					LDAP_XSTRING(mdb_tool_entry_reindex)
+					": (Truncate) mdb_drop(%s) failed: %s (%d)\n",
+					mi->mi_attrs[i]->ai_desc->ad_type->sat_cname.bv_val,
+					mdb_strerror(rc), rc );
+				return -1;
+			}
+		}
+		slapMode ^= SLAP_TRUNCATE_MODE;
+	}
+
 	/*
 	 * just (re)add them for now
-	 * assume that some other routine (not yet implemented)
-	 * will zap index databases
-	 *
+	 * Use truncate mode to empty/reset index databases
 	 */
 
 	Debug( LDAP_DEBUG_TRACE,
-		"=> " LDAP_XSTRING(mdb_tool_entry_reindex) "( %ld, \"%s\" )\n",
-		(long) id, e->e_dn, 0 );
+		"=> " LDAP_XSTRING(mdb_tool_entry_reindex) "( %ld )\n",
+		(long) id, 0, 0 );
 
 	op.o_hdr = &ohdr;
 	op.o_bd = be;
@@ -769,8 +840,12 @@ done:
 	if( rc == 0 ) {
 		mdb_writes++;
 		if ( mdb_writes >= mdb_writes_per_commit ) {
-			mdb_tool_idl_flush( be, txi );
+			unsigned i;
+			MDB_TOOL_IDL_FLUSH( be, txi );
 			rc = mdb_txn_commit( txi );
+			mdb_writes = 0;
+			for ( i=0; i<mi->mi_nattrs; i++ )
+				mi->mi_attrs[i]->ai_cursor = NULL;
 			if( rc != 0 ) {
 				Debug( LDAP_DEBUG_ANY,
 					"=> " LDAP_XSTRING(mdb_tool_entry_reindex)
@@ -782,7 +857,11 @@ done:
 		}
 
 	} else {
+		unsigned i;
+		mdb_writes = 0;
 		mdb_txn_abort( txi );
+		for ( i=0; i<mi->mi_nattrs; i++ )
+			mi->mi_attrs[i]->ai_cursor = NULL;
 		Debug( LDAP_DEBUG_ANY,
 			"=> " LDAP_XSTRING(mdb_tool_entry_reindex)
 			": txn_aborted! err=%d\n",
@@ -802,7 +881,6 @@ ID mdb_tool_entry_modify(
 {
 	int rc;
 	struct mdb_info *mdb;
-	MDB_txn *tid;
 	Operation op = {0};
 	Opheader ohdr = {0};
 
@@ -825,15 +903,17 @@ ID mdb_tool_entry_modify(
 		mdb_cursor_close( cursor );
 		cursor = NULL;
 	}
-	rc = mdb_txn_begin( mdb->mi_dbenv, NULL, 0, &tid );
-	if( rc != 0 ) {
-		snprintf( text->bv_val, text->bv_len,
-			"txn_begin failed: %s (%d)",
-			mdb_strerror(rc), rc );
-		Debug( LDAP_DEBUG_ANY,
-			"=> " LDAP_XSTRING(mdb_tool_entry_modify) ": %s\n",
-			 text->bv_val, 0, 0 );
-		return NOID;
+	if ( !txn ) {
+		rc = mdb_txn_begin( mdb->mi_dbenv, NULL, 0, &txn );
+		if( rc != 0 ) {
+			snprintf( text->bv_val, text->bv_len,
+				"txn_begin failed: %s (%d)",
+				mdb_strerror(rc), rc );
+			Debug( LDAP_DEBUG_ANY,
+				"=> " LDAP_XSTRING(mdb_tool_entry_modify) ": %s\n",
+				 text->bv_val, 0, 0 );
+			return NOID;
+		}
 	}
 
 	op.o_hdr = &ohdr;
@@ -842,7 +922,7 @@ ID mdb_tool_entry_modify(
 	op.o_tmpmfuncs = &ch_mfuncs;
 
 	/* id2entry index */
-	rc = mdb_id2entry_update( &op, tid, e );
+	rc = mdb_id2entry_update( &op, txn, NULL, e );
 	if( rc != 0 ) {
 		snprintf( text->bv_val, text->bv_len,
 				"id2entry_update failed: err=%d", rc );
@@ -854,7 +934,7 @@ ID mdb_tool_entry_modify(
 
 done:
 	if( rc == 0 ) {
-		rc = mdb_txn_commit( tid );
+		rc = mdb_txn_commit( txn );
 		if( rc != 0 ) {
 			snprintf( text->bv_val, text->bv_len,
 					"txn_commit failed: %s (%d)",
@@ -866,7 +946,7 @@ done:
 		}
 
 	} else {
-		mdb_txn_abort( tid );
+		mdb_txn_abort( txn );
 		snprintf( text->bv_val, text->bv_len,
 			"txn_aborted! %s (%d)",
 			mdb_strerror(rc), rc );
@@ -875,11 +955,11 @@ done:
 			text->bv_val, 0, 0 );
 		e->e_id = NOID;
 	}
+	txn = NULL;
 
 	return e->e_id;
 }
 
-#if 0
 static void *
 mdb_tool_index_task( void *ctx, void *ptr )
 {
@@ -901,15 +981,15 @@ mdb_tool_index_task( void *ctx, void *ptr )
 			break;
 		}
 		ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex );
-
-		mdb_tool_index_threads[base] = mdb_index_recrun( mdb_tool_ix_op,
+		mdb_tool_index_rec[base].ir_i = mdb_index_recrun( mdb_tool_ix_op,
+			mdb_tool_ix_txn,
 			mdb_tool_info, mdb_tool_index_rec, mdb_tool_ix_id, base );
 	}
 
 	return NULL;
 }
-#endif
 
+#ifdef MDB_TOOL_IDL_CACHING
 static int
 mdb_tool_idl_cmp( const void *v1, const void *v2 )
 {
@@ -921,10 +1001,10 @@ mdb_tool_idl_cmp( const void *v1, const void *v2 )
 }
 
 static int
-mdb_tool_idl_flush_one( MDB_cursor *mc, mdb_tool_idl_cache *ic )
+mdb_tool_idl_flush_one( MDB_cursor *mc, AttrInfo *ai, mdb_tool_idl_cache *ic )
 {
 	mdb_tool_idl_cache_entry *ice;
-	MDB_val key, data;
+	MDB_val key, data[2];
 	int i, rc;
 	ID id, nid;
 
@@ -936,54 +1016,56 @@ mdb_tool_idl_flush_one( MDB_cursor *mc, mdb_tool_idl_cache *ic )
 	key.mv_data = ic->kstr.bv_val;
 	key.mv_size = ic->kstr.bv_len;
 
-	rc = mdb_cursor_get( mc, &key, &data, MDB_SET );
-	/* If key already exists and we're writing a range... */
-	if ( rc == 0 && ic->count > MDB_IDL_DB_SIZE ) {
-		nid = *(ID *)data.mv_data;
-		/* If it's not currently a range, must delete old info */
-		if ( nid ) {
-			ic->first = nid;
-			mdb_cursor_del( mc, MDB_NODUPDATA );
-
-			goto setrange;
-		} else {
-			/* Skip lo */
-			rc = mdb_cursor_get( mc, &key, &data, MDB_NEXT_DUP );
+	if ( ic->count > MDB_IDL_DB_SIZE ) {
+		while ( ic->flags & WAS_FOUND ) {
+			rc = mdb_cursor_get( mc, &key, data, MDB_SET );
+			if ( rc ) {
+				/* FIXME: find out why this happens */
+				ic->flags = 0;
+				break;
+			}
+			if ( ic->flags & WAS_RANGE ) {
+				/* Skip lo */
+				rc = mdb_cursor_get( mc, &key, data, MDB_NEXT_DUP );
 
-			/* Get hi */
-			rc = mdb_cursor_get( mc, &key, &data, MDB_NEXT_DUP );
+				/* Get hi */
+				rc = mdb_cursor_get( mc, &key, data, MDB_NEXT_DUP );
 
-			/* Store range hi */
-			data.mv_data = &ic->last;
-			rc = mdb_cursor_put( mc, &key, &data, MDB_CURRENT );
+				/* Store range hi */
+				data[0].mv_data = &ic->last;
+				rc = mdb_cursor_put( mc, &key, data, MDB_CURRENT );
+			} else {
+				/* Delete old data, replace with range */
+				ic->first = *(ID *)data[0].mv_data;
+				mdb_cursor_del( mc, MDB_NODUPDATA );
+			}
+			break;
 		}
-		rc = 0;
-	} else if ( rc && rc != MDB_NOTFOUND ) {
-		rc = -1;
-	} else if ( ic->count > MDB_IDL_DB_SIZE ) {
-		/* range, didn't exist before */
-setrange:
-		nid = 0;
-		data.mv_size = sizeof(ID);
-		data.mv_data = &nid;
-		rc = mdb_cursor_put( mc, &key, &data, 0 );
-		if ( rc == 0 ) {
-			data.mv_data = &ic->first;
-			rc = mdb_cursor_put( mc, &key, &data, 0 );
+		if ( !(ic->flags & WAS_RANGE)) {
+			/* range, didn't exist before */
+			nid = 0;
+			data[0].mv_size = sizeof(ID);
+			data[0].mv_data = &nid;
+			rc = mdb_cursor_put( mc, &key, data, 0 );
 			if ( rc == 0 ) {
-				data.mv_data = &ic->last;
-				rc = mdb_cursor_put( mc, &key, &data, 0 );
+				data[0].mv_data = &ic->first;
+				rc = mdb_cursor_put( mc, &key, data, 0 );
+				if ( rc == 0 ) {
+					data[0].mv_data = &ic->last;
+					rc = mdb_cursor_put( mc, &key, data, 0 );
+				}
+			}
+			if ( rc ) {
+				rc = -1;
 			}
-		}
-		if ( rc ) {
-			rc = -1;
 		}
 	} else {
+		/* Normal write */
 		int n;
 
-		data.mv_size = sizeof(ID);
-		/* Just a normal write */
+		data[0].mv_size = sizeof(ID);
 		rc = 0;
+		i = ic->offset;
 		for ( ice = ic->head, n=0; ice; ice = ice->next, n++ ) {
 			int end;
 			if ( ice->next ) {
@@ -993,43 +1075,40 @@ setrange:
 				if ( !end )
 					end = IDBLOCK;
 			}
-			for ( i=0; i<end; i++ ) {
-				if ( !ice->ids[i] ) continue;
-				data.mv_data = &ice->ids[i];
-				rc = mdb_cursor_put( mc, &key, &data, MDB_NODUPDATA|MDB_APPEND );
-				if ( rc ) {
-					if ( rc == MDB_KEYEXIST ) {
-						rc = 0;
-						continue;
-					}
-					rc = -1;
-					break;
-				}
-			}
+			data[1].mv_size = end - i;
+			data[0].mv_data = &ice->ids[i];
+			i = 0;
+			rc = mdb_cursor_put( mc, &key, data, MDB_NODUPDATA|MDB_APPEND|MDB_MULTIPLE );
 			if ( rc ) {
+				if ( rc == MDB_KEYEXIST ) {
+					rc = 0;
+					continue;
+				}
 				rc = -1;
 				break;
 			}
 		}
 		if ( ic->head ) {
-			ic->tail->next = mdb_tool_idl_free_list;
-			mdb_tool_idl_free_list = ic->head;
+			ic->tail->next = ai->ai_flist;
+			ai->ai_flist = ic->head;
 		}
 	}
-	ch_free( ic );
+	ic->head = ai->ai_clist;
+	ai->ai_clist = ic;
 	return rc;
 }
 
 static int
-mdb_tool_idl_flush_db( MDB_txn *txn, MDB_dbi dbi, Avlnode *root )
+mdb_tool_idl_flush_db( MDB_txn *txn, AttrInfo *ai )
 {
 	MDB_cursor *mc;
+	Avlnode *root;
 	int rc;
 
-	mdb_cursor_open( txn, dbi, &mc );
-	root = tavl_end( root, TAVL_DIR_LEFT );
+	mdb_cursor_open( txn, ai->ai_dbi, &mc );
+	root = tavl_end( ai->ai_root, TAVL_DIR_LEFT );
 	do {
-		rc = mdb_tool_idl_flush_one( mc, root->avl_data );
+		rc = mdb_tool_idl_flush_one( mc, ai, root->avl_data );
 		if ( rc != -1 )
 			rc = 0;
 	} while ((root = tavl_next(root, TAVL_DIR_RIGHT)));
@@ -1043,13 +1122,13 @@ mdb_tool_idl_flush( BackendDB *be, MDB_txn *txn )
 {
 	struct mdb_info *mdb = (struct mdb_info *) be->be_private;
 	int rc = 0;
-	unsigned int i;
+	unsigned int i, dbi;
 
-	for ( i=MDB_NDB; i < mdb->mi_nattrs+MDB_NDB; i++ ) {
-		if ( !mdb_tool_roots[i] ) continue;
-		rc = mdb_tool_idl_flush_db( txn, i, mdb_tool_roots[i] );
-		tavl_free(mdb_tool_roots[i], NULL);
-		mdb_tool_roots[i] = NULL;
+	for ( i=0; i < mdb->mi_nattrs; i++ ) {
+		if ( !mdb->mi_attrs[i]->ai_root ) continue;
+		rc = mdb_tool_idl_flush_db( txn, mdb->mi_attrs[i] );
+		tavl_free(mdb->mi_attrs[i]->ai_root, NULL);
+		mdb->mi_attrs[i]->ai_root = NULL;
 		if ( rc )
 			break;
 	}
@@ -1064,12 +1143,14 @@ int mdb_tool_idl_add(
 	MDB_dbi dbi;
 	mdb_tool_idl_cache *ic, itmp;
 	mdb_tool_idl_cache_entry *ice;
-	int i, rc;
+	int i, rc, lcount;
+	AttrInfo *ai = (AttrInfo *)mc;
+	mc = ai->ai_cursor;
 
-	dbi = mdb_cursor_dbi(mc);
+	dbi = ai->ai_dbi;
 	for (i=0; keys[i].bv_val; i++) {
 	itmp.kstr = keys[i];
-	ic = tavl_find( (Avlnode *)mdb_tool_roots[dbi], &itmp, mdb_tool_idl_cmp );
+	ic = tavl_find( (Avlnode *)ai->ai_root, &itmp, mdb_tool_idl_cmp );
 
 	/* No entry yet, create one */
 	if ( !ic ) {
@@ -1077,31 +1158,40 @@ int mdb_tool_idl_add(
 		ID nid;
 		int rc;
 
-		ic = ch_malloc( sizeof( mdb_tool_idl_cache ) + itmp.kstr.bv_len );
+		if ( ai->ai_clist ) {
+			ic = ai->ai_clist;
+			ai->ai_clist = ic->head;
+		} else {
+			ic = ch_malloc( sizeof( mdb_tool_idl_cache ) + itmp.kstr.bv_len + 4 );
+		}
 		ic->kstr.bv_len = itmp.kstr.bv_len;
 		ic->kstr.bv_val = (char *)(ic+1);
 		memcpy( ic->kstr.bv_val, itmp.kstr.bv_val, ic->kstr.bv_len );
 		ic->head = ic->tail = NULL;
 		ic->last = 0;
 		ic->count = 0;
-		tavl_insert( (Avlnode **)&mdb_tool_roots[dbi], ic, mdb_tool_idl_cmp,
+		ic->offset = 0;
+		ic->flags = 0;
+		tavl_insert( (Avlnode **)&ai->ai_root, ic, mdb_tool_idl_cmp,
 			avl_dup_error );
 
 		/* load existing key count here */
 		key.mv_size = keys[i].bv_len;
 		key.mv_data = keys[i].bv_val;
-		data.mv_size = sizeof( ID );
-		data.mv_data = &nid;
 		rc = mdb_cursor_get( mc, &key, &data, MDB_SET );
 		if ( rc == 0 ) {
+			ic->flags |= WAS_FOUND;
+			nid = *(ID *)data.mv_data;
 			if ( nid == 0 ) {
 				ic->count = MDB_IDL_DB_SIZE+1;
+				ic->flags |= WAS_RANGE;
 			} else {
 				size_t count;
 
 				mdb_cursor_count( mc, &count );
 				ic->count = count;
 				ic->first = nid;
+				ic->offset = count & (IDBLOCK-1);
 			}
 		}
 	}
@@ -1111,12 +1201,9 @@ int mdb_tool_idl_add(
 		continue;
 	/* Are we at the limit, and converting to a range? */
 	} else if ( ic->count == MDB_IDL_DB_SIZE ) {
-		int n;
-		for ( ice = ic->head, n=0; ice; ice = ice->next, n++ )
-			/* counting */ ;
-		if ( n ) {
-			ic->tail->next = mdb_tool_idl_free_list;
-			mdb_tool_idl_free_list = ic->head;
+		if ( ic->head ) {
+			ic->tail->next = ai->ai_flist;
+			ai->ai_flist = ic->head;
 		}
 		ic->head = ic->tail = NULL;
 		ic->last = id;
@@ -1124,29 +1211,32 @@ int mdb_tool_idl_add(
 		continue;
 	}
 	/* No free block, create that too */
-	if ( !ic->tail || ( ic->count & (IDBLOCK-1)) == 0) {
-		ice = NULL;
-		if ( mdb_tool_idl_free_list ) {
-			ice = mdb_tool_idl_free_list;
-			mdb_tool_idl_free_list = ice->next;
-		}
-		if ( !ice ) {
+	lcount = ic->count & (IDBLOCK-1);
+	if ( !ic->tail || lcount == 0) {
+		if ( ai->ai_flist ) {
+			ice = ai->ai_flist;
+			ai->ai_flist = ice->next;
+		} else {
 			ice = ch_malloc( sizeof( mdb_tool_idl_cache_entry ));
 		}
-		memset( ice, 0, sizeof( *ice ));
+		ice->next = NULL;
 		if ( !ic->head ) {
 			ic->head = ice;
 		} else {
 			ic->tail->next = ice;
 		}
 		ic->tail = ice;
+		if ( lcount )
+			ice->ids[lcount-1] = 0;
 		if ( !ic->count )
 			ic->first = id;
 	}
 	ice = ic->tail;
-	ice->ids[ ic->count & (IDBLOCK-1) ] = id;
+	if (!lcount || ice->ids[lcount-1] != id)
+		ice->ids[lcount] = id;
 	ic->count++;
 	}
 
 	return 0;
 }
+#endif /* MDB_TOOL_IDL_CACHING */