X-Git-Url: https://git.sur5r.net/?a=blobdiff_plain;f=servers%2Fslapd%2Fback-bdb%2Ftools.c;h=167f3c7e5e2b46d198a5e2114a5cba5933a87cf3;hb=0af1940f3fb59fe57b2281ef253fe1341c505c2c;hp=354589cc35b53012b21577b3574a3796256a7276;hpb=9459a7f53589f5939bc86a4c20baab7c08b8de84;p=openldap

diff --git a/servers/slapd/back-bdb/tools.c b/servers/slapd/back-bdb/tools.c
index 354589cc35..167f3c7e5e 100644
--- a/servers/slapd/back-bdb/tools.c
+++ b/servers/slapd/back-bdb/tools.c
@@ -1,8 +1,17 @@
 /* tools.c - tools for slap tools */
 /* $OpenLDAP$ */
-/*
- * Copyright 1998-2003 The OpenLDAP Foundation, All Rights Reserved.
- * COPYING RESTRICTIONS APPLY, see COPYRIGHT file
+/* This work is part of OpenLDAP Software <http://www.openldap.org/>.
+ *
+ * Copyright 2000-2006 The OpenLDAP Foundation.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the OpenLDAP
+ * Public License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * top-level directory of the distribution or, alternatively, at
+ * <http://www.OpenLDAP.org/license.html>.
  */
 
 #include "portable.h"
@@ -10,8 +19,9 @@
 #include <stdio.h>
 #include <ac/string.h>
 
+#define AVL_INTERNAL
 #include "back-bdb.h"
-#include "external.h"
+#include "idl.h"
 
 static DBC *cursor = NULL;
 static DBT key, data;
@@ -26,22 +36,95 @@ static dn_id hbuf[HOLE_SIZE], *holes = hbuf;
 static unsigned nhmax = HOLE_SIZE;
 static unsigned nholes;
 
+static int index_nattrs;
+
+#ifdef BDB_TOOL_IDL_CACHING
+#define bdb_tool_idl_cmp		BDB_SYMBOL(tool_idl_cmp)
+#define bdb_tool_idl_flush_one		BDB_SYMBOL(tool_idl_flush_one)
+#define bdb_tool_idl_flush		BDB_SYMBOL(tool_idl_flush)
+
+static int bdb_tool_idl_flush( BackendDB *be );
+
+#define	IDBLOCK	1024
+
+typedef struct bdb_tool_idl_cache_entry {
+	struct bdb_tool_idl_cache_entry *next;
+	ID ids[IDBLOCK];
+} bdb_tool_idl_cache_entry;
+ 
+typedef struct bdb_tool_idl_cache {
+	struct berval kstr;
+	bdb_tool_idl_cache_entry *head, *tail;
+	ID first, last;
+	int count;
+} bdb_tool_idl_cache;
+
+static bdb_tool_idl_cache_entry *bdb_tool_idl_free_list;
+#endif	/* BDB_TOOL_IDL_CACHING */
+
+static ID bdb_tool_ix_id;
+static Operation *bdb_tool_ix_op;
+static int *bdb_tool_index_threads, bdb_tool_index_tcount;
+static void *bdb_tool_index_rec;
+static struct bdb_info *bdb_tool_info;
+static ldap_pvt_thread_mutex_t bdb_tool_index_mutex;
+static ldap_pvt_thread_cond_t bdb_tool_index_cond;
+
+static void * bdb_tool_index_task( void *ctx, void *ptr );
+
 int bdb_tool_entry_open(
 	BackendDB *be, int mode )
 {
+	struct bdb_info *bdb = (struct bdb_info *) be->be_private;
+
 	/* initialize key and data thangs */
 	DBTzero( &key );
 	DBTzero( &data );
 	key.flags = DB_DBT_REALLOC;
 	data.flags = DB_DBT_REALLOC;
 
+	if (cursor == NULL) {
+		int rc = bdb->bi_id2entry->bdi_db->cursor(
+			bdb->bi_id2entry->bdi_db, NULL, &cursor,
+			bdb->bi_db_opflags );
+		if( rc != 0 ) {
+			return -1;
+		}
+	}
+
+	/* Set up for threaded slapindex */
+	if (( slapMode & (SLAP_TOOL_QUICK|SLAP_TOOL_READONLY)) == SLAP_TOOL_QUICK
+		&& bdb->bi_nattrs ) {
+		if ( !bdb_tool_info ) {
+			int i;
+			ldap_pvt_thread_mutex_init( &bdb_tool_index_mutex );
+			ldap_pvt_thread_cond_init( &bdb_tool_index_cond );
+			bdb_tool_index_threads = ch_malloc( slap_tool_thread_max * sizeof( int ));
+			bdb_tool_index_rec = ch_malloc( bdb->bi_nattrs * sizeof( IndexRec ));
+			bdb_tool_index_tcount = slap_tool_thread_max - 1;
+			for (i=1; i<slap_tool_thread_max; i++) {
+				int *ptr = ch_malloc( sizeof( int ));
+				*ptr = i;
+				ldap_pvt_thread_pool_submit( &connection_pool,
+					bdb_tool_index_task, ptr );
+			}
+		}
+		bdb_tool_info = bdb;
+	}
+
 	return 0;
 }
 
 int bdb_tool_entry_close(
 	BackendDB *be )
 {
-	assert( be != NULL );
+	if ( bdb_tool_info ) {
+		slapd_shutdown = 1;
+		ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex );
+		bdb_tool_index_tcount = slap_tool_thread_max - 1;
+		ldap_pvt_thread_cond_broadcast( &bdb_tool_index_cond );
+		ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex );
+	}
 
 	if( key.data ) {
 		ch_free( key.data );
@@ -57,6 +140,10 @@ int bdb_tool_entry_close(
 		cursor = NULL;
 	}
 
+#ifdef BDB_TOOL_IDL_CACHING
+	bdb_tool_idl_flush( be );
+#endif
+
 	if( nholes ) {
 		unsigned i;
 		fprintf( stderr, "Error, entries missing!\n");
@@ -81,26 +168,31 @@ ID bdb_tool_entry_next(
 	assert( slapMode & SLAP_TOOL_MODE );
 	assert( bdb != NULL );
 	
-	if (cursor == NULL) {
-		rc = bdb->bi_id2entry->bdi_db->cursor(
-			bdb->bi_id2entry->bdi_db, NULL, &cursor,
-			bdb->bi_db_opflags );
-		if( rc != 0 ) {
-			return NOID;
-		}
-	}
-
 	rc = cursor->c_get( cursor, &key, &data, DB_NEXT );
 
 	if( rc != 0 ) {
-		return NOID;
+		/* If we're doing linear indexing and there are more attrs to
+		 * index, and we're at the end of the database, start over.
+		 */
+		if ( index_nattrs && rc == DB_NOTFOUND ) {
+			/* optional - do a checkpoint here? */
+			bdb_attr_info_free( bdb->bi_attrs[0] );
+			bdb->bi_attrs[0] = bdb->bi_attrs[index_nattrs];
+			index_nattrs--;
+			rc = cursor->c_get( cursor, &key, &data, DB_FIRST );
+			if ( rc ) {
+				return NOID;
+			}
+		} else {
+			return NOID;
+		}
 	}
 
 	if( data.data == NULL ) {
 		return NOID;
 	}
 
-	AC_MEMCPY( &id, key.data, key.size );
+	BDB_DISK2ID( key.data, &id );
 	return id;
 }
 
@@ -109,38 +201,25 @@ ID bdb_tool_dn2id_get(
 	struct berval *dn
 )
 {
-	struct bdb_info *bdb = (struct bdb_info *) be->be_private;
-	DB *db = bdb->bi_dn2id->bdi_db;
+	Operation op = {0};
+	Opheader ohdr = {0};
+	EntryInfo *ei = NULL;
 	int rc;
-	DBT	key, data;
-	ID	id;
 
-	DBTzero( &key );
-	key.size = dn->bv_len + 2;
-	key.data = ch_malloc( key.size );
-	((char*)key.data)[0] = DN_BASE_PREFIX;
-	AC_MEMCPY( &((char*)key.data)[1], dn->bv_val, key.size - 1 );
+	if ( BER_BVISEMPTY(dn) )
+		return 0;
 
-	DBTzero( &data );
-	data.data = &id;
-	data.ulen = sizeof(ID);
-	data.flags = DB_DBT_USERMEM;
-
-	rc = db->get( db, NULL, &key, &data, bdb->bi_db_opflags );
-
-    if( rc != 0 ) {
-#ifdef NEW_LOGGING
-		LDAP_LOG ( INDEX, ERR, "bdb_tool_dn2id_get: get failed %s (%d)\n",
-				db_strerror(rc), rc, 0 );
-#else
-		Debug( LDAP_DEBUG_TRACE, "bdb_tool_dn2id_get: get failed: %s (%d)\n",
-				db_strerror( rc ), rc, 0 );
-#endif
-		id = NOID;
-	}
+	op.o_hdr = &ohdr;
+	op.o_bd = be;
+	op.o_tmpmemctx = NULL;
+	op.o_tmpmfuncs = &ch_mfuncs;
 
-	ch_free( key.data );
-	return id;
+	rc = bdb_cache_find_ndn( &op, NULL, dn, &ei );
+	if ( ei ) bdb_cache_entryinfo_unlock( ei );
+	if ( rc == DB_NOTFOUND )
+		return NOID;
+	
+	return ei->bei_id;
 }
 
 int bdb_tool_id2entry_get(
@@ -149,7 +228,20 @@ int bdb_tool_id2entry_get(
 	Entry **e
 )
 {
-	return bdb_id2entry( be, NULL, id, e );
+	int rc = bdb_id2entry( be, NULL, 0, id, e );
+
+	if ( rc == DB_NOTFOUND && id == 0 ) {
+		Entry *dummy = ch_calloc( 1, sizeof(Entry) );
+		struct berval gluebv = BER_BVC("glue");
+		dummy->e_name.bv_val = ch_strdup( "" );
+		dummy->e_nname.bv_val = ch_strdup( "" );
+		attr_merge_one( dummy, slap_schema.si_ad_objectClass, &gluebv, NULL );
+		attr_merge_one( dummy, slap_schema.si_ad_structuralObjectClass,
+			&gluebv, NULL );
+		*e = dummy;
+		rc = LDAP_SUCCESS;
+	}
+	return rc;
 }
 
 Entry* bdb_tool_entry_get( BackendDB *be, ID id )
@@ -162,26 +254,38 @@ Entry* bdb_tool_entry_get( BackendDB *be, ID id )
 	assert( slapMode & SLAP_TOOL_MODE );
 	assert( data.data != NULL );
 
-#ifndef BDB_HIER
 	DBT2bv( &data, &bv );
 
+#ifdef SLAP_ZONE_ALLOC
+	/* FIXME: will add ctx later */
+	rc = entry_decode( &bv, &e, NULL );
+#else
 	rc = entry_decode( &bv, &e );
+#endif
 
 	if( rc == LDAP_SUCCESS ) {
 		e->e_id = id;
 	}
-#else
-	{
+#ifdef BDB_HIER
+	if ( slapMode & SLAP_TOOL_READONLY ) {
 		EntryInfo *ei = NULL;
 		Operation op = {0};
+		Opheader ohdr = {0};
 
+		op.o_hdr = &ohdr;
 		op.o_bd = be;
 		op.o_tmpmemctx = NULL;
 		op.o_tmpmfuncs = &ch_mfuncs;
 
-		rc = bdb_cache_find_id( &op, NULL, id, &ei, 0, 0, NULL );
-		if ( rc == LDAP_SUCCESS )
-			e = ei->bei_e;
+		rc = bdb_cache_find_parent( &op, NULL, cursor->locker, id, &ei );
+		if ( rc == LDAP_SUCCESS ) {
+			bdb_cache_entryinfo_unlock( ei );
+			e->e_private = ei;
+			ei->bei_e = e;
+			bdb_fix_dn( e, 0 );
+			ei->bei_e = NULL;
+			e->e_private = NULL;
+		}
 	}
 #endif
 	return e;
@@ -192,71 +296,77 @@ static int bdb_tool_next_id(
 	DB_TXN *tid,
 	Entry *e,
 	struct berval *text,
-	int hole,
-	u_int32_t locker )
+	int hole )
 {
-	struct bdb_info *bdb = (struct bdb_info *) op->o_bd->be_private;
-	struct berval dn = e->e_nname;
-	struct berval pdn;
-	EntryInfo *ei = NULL;
+	struct berval dn = e->e_name;
+	struct berval ndn = e->e_nname;
+	struct berval pdn, npdn;
+	EntryInfo *ei = NULL, eidummy;
 	int rc;
 
-	rc = bdb_cache_find_ndn( op, tid, &dn, &ei, locker );
+	if (ndn.bv_len == 0) {
+		e->e_id = 0;
+		return 0;
+	}
+
+	rc = bdb_cache_find_ndn( op, tid, &ndn, &ei );
 	if ( ei ) bdb_cache_entryinfo_unlock( ei );
 	if ( rc == DB_NOTFOUND ) {
-		if ( be_issuffix( op->o_bd, &dn ) ) {
-			pdn = slap_empty_bv;
-		} else {
+		if ( !be_issuffix( op->o_bd, &ndn ) ) {
+			ID eid = e->e_id;
 			dnParent( &dn, &pdn );
-			e->e_nname = pdn;
-			rc = bdb_tool_next_id( op, tid, e, text, 1, locker );
+			dnParent( &ndn, &npdn );
+			e->e_name = pdn;
+			e->e_nname = npdn;
+			rc = bdb_tool_next_id( op, tid, e, text, 1 );
+			e->e_name = dn;
+			e->e_nname = ndn;
 			if ( rc ) {
 				return rc;
 			}
+			/* If parent didn't exist, it was created just now
+			 * and its ID is now in e->e_id. Make sure the current
+			 * entry gets added under the new parent ID.
+			 */
+			if ( eid != e->e_id ) {
+				eidummy.bei_id = e->e_id;
+				ei = &eidummy;
+			}
 		}
 		rc = bdb_next_id( op->o_bd, tid, &e->e_id );
 		if ( rc ) {
 			snprintf( text->bv_val, text->bv_len,
 				"next_id failed: %s (%d)",
 				db_strerror(rc), rc );
-#ifdef NEW_LOGGING
-		LDAP_LOG ( TOOLS, ERR, 
-			"=> bdb_tool_next_id: %s\n", text->bv_val, 0, 0 );
-#else
 		Debug( LDAP_DEBUG_ANY,
 			"=> bdb_tool_next_id: %s\n", text->bv_val, 0, 0 );
-#endif
 			return rc;
 		}
-		e->e_nname = dn;
 		rc = bdb_dn2id_add( op, tid, ei, e );
 		if ( rc ) {
 			snprintf( text->bv_val, text->bv_len, 
 				"dn2id_add failed: %s (%d)",
 				db_strerror(rc), rc );
-#ifdef NEW_LOGGING
-		LDAP_LOG ( TOOLS, ERR, 
-			"=> bdb_tool_next_id: %s\n", text->bv_val, 0, 0 );
-#else
 		Debug( LDAP_DEBUG_ANY,
 			"=> bdb_tool_next_id: %s\n", text->bv_val, 0, 0 );
-#endif
 		} else if ( hole ) {
 			if ( nholes == nhmax - 1 ) {
 				if ( holes == hbuf ) {
-					holes = ch_malloc( nhmax * sizeof(ID) * 2 );
+					holes = ch_malloc( nhmax * sizeof(dn_id) * 2 );
 					AC_MEMCPY( holes, hbuf, sizeof(hbuf) );
 				} else {
-					holes = ch_realloc( holes, nhmax * sizeof(ID) * 2 );
+					holes = ch_realloc( holes, nhmax * sizeof(dn_id) * 2 );
 				}
 				nhmax *= 2;
 			}
-			ber_dupbv( &holes[nholes].dn, &dn );
+			ber_dupbv( &holes[nholes].dn, &ndn );
 			holes[nholes++].id = e->e_id;
 		}
 	} else if ( !hole ) {
 		unsigned i;
 
+		e->e_id = ei->bei_id;
+
 		for ( i=0; i<nholes; i++) {
 			if ( holes[i].id == e->e_id ) {
 				int j;
@@ -273,6 +383,60 @@ static int bdb_tool_next_id(
 	return rc;
 }
 
+static int
+bdb_tool_index_add(
+	Operation *op,
+	DB_TXN *txn,
+	Entry *e )
+{
+	struct bdb_info *bdb = (struct bdb_info *) op->o_bd->be_private;
+
+	if ( slapMode & SLAP_TOOL_QUICK ) {
+		IndexRec *ir;
+		int i, rc;
+		Attribute *a;
+		
+		ir = bdb_tool_index_rec;
+		memset(ir, 0, bdb->bi_nattrs * sizeof( IndexRec ));
+
+		for ( a = e->e_attrs; a != NULL; a = a->a_next ) {
+			rc = bdb_index_recset( bdb, a, a->a_desc->ad_type, 
+				&a->a_desc->ad_tags, ir );
+			if ( rc )
+				return rc;
+		}
+		bdb_tool_ix_id = e->e_id;
+		bdb_tool_ix_op = op;
+		ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex );
+		/* Wait for all threads to be ready */
+		while ( bdb_tool_index_tcount ) {
+			ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex );
+			ldap_pvt_thread_yield();
+			ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex );
+		}
+		for ( i=1; i<slap_tool_thread_max; i++ )
+			bdb_tool_index_threads[i] = LDAP_BUSY;
+		bdb_tool_index_tcount = slap_tool_thread_max - 1;
+		ldap_pvt_thread_cond_broadcast( &bdb_tool_index_cond );
+		ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex );
+		rc = bdb_index_recrun( op, bdb, ir, e->e_id, 0 );
+		if ( rc )
+			return rc;
+		for ( i=1; i<slap_tool_thread_max; i++ ) {
+			if ( bdb_tool_index_threads[i] == LDAP_BUSY ) {
+				ldap_pvt_thread_yield();
+				i--;
+				continue;
+			}
+			if ( bdb_tool_index_threads[i] )
+				return bdb_tool_index_threads[i];
+		}
+		return 0;
+	} else {
+		return bdb_index_entry_add( op, txn, e );
+	}
+}
+
 ID bdb_tool_entry_put(
 	BackendDB *be,
 	Entry *e,
@@ -282,112 +446,92 @@ ID bdb_tool_entry_put(
 	struct bdb_info *bdb = (struct bdb_info *) be->be_private;
 	DB_TXN *tid = NULL;
 	Operation op = {0};
-	u_int32_t locker;
+	Opheader ohdr = {0};
 
 	assert( be != NULL );
 	assert( slapMode & SLAP_TOOL_MODE );
 
-	assert( text );
-	assert( text->bv_val );
+	assert( text != NULL );
+	assert( text->bv_val != NULL );
 	assert( text->bv_val[0] == '\0' );	/* overconservative? */
 
-#ifdef NEW_LOGGING
-	LDAP_LOG ( TOOLS, ARGS, "=> bdb_tool_entry_put( %ld, \"%s\" )\n",
-		(long) e->e_id, e->e_dn, 0 );
-#else
-	Debug( LDAP_DEBUG_TRACE, "=> bdb_tool_entry_put( %ld, \"%s\" )\n",
-		(long) e->e_id, e->e_dn, 0 );
-#endif
+	Debug( LDAP_DEBUG_TRACE, "=> " LDAP_XSTRING(bdb_tool_entry_put)
+		"( %ld, \"%s\" )\n", (long) e->e_id, e->e_dn, 0 );
 
+	if (! (slapMode & SLAP_TOOL_QUICK)) {
 	rc = TXN_BEGIN( bdb->bi_dbenv, NULL, &tid, 
 		bdb->bi_db_opflags );
 	if( rc != 0 ) {
 		snprintf( text->bv_val, text->bv_len,
 			"txn_begin failed: %s (%d)",
 			db_strerror(rc), rc );
-#ifdef NEW_LOGGING
-	LDAP_LOG ( TOOLS, ERR, "=> bdb_tool_entry_put: %s\n", text->bv_val, 0, 0 );
-#else
 		Debug( LDAP_DEBUG_ANY,
-			"=> bdb_tool_entry_put: %s\n",
+			"=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
 			 text->bv_val, 0, 0 );
-#endif
 		return NOID;
 	}
+	}
 
+	op.o_hdr = &ohdr;
 	op.o_bd = be;
 	op.o_tmpmemctx = NULL;
 	op.o_tmpmfuncs = &ch_mfuncs;
 
-	locker = TXN_ID( tid );
 	/* add dn2id indices */
-	rc = bdb_tool_next_id( &op, tid, e, text, 0, locker );
+	rc = bdb_tool_next_id( &op, tid, e, text, 0 );
 	if( rc != 0 ) {
 		goto done;
 	}
 
-	/* id2entry index */
-	rc = bdb_id2entry_add( be, tid, e );
+	if ( !bdb->bi_linear_index )
+		rc = bdb_tool_index_add( &op, tid, e );
 	if( rc != 0 ) {
 		snprintf( text->bv_val, text->bv_len,
-				"id2entry_add failed: %s (%d)",
+				"index_entry_add failed: %s (%d)",
 				db_strerror(rc), rc );
-#ifdef NEW_LOGGING
-		LDAP_LOG ( TOOLS, ERR, 
-			"=> bdb_tool_entry_put: %s\n", text->bv_val, 0, 0 );
-#else
 		Debug( LDAP_DEBUG_ANY,
-			"=> bdb_tool_entry_put: %s\n", text->bv_val, 0, 0 );
-#endif
+			"=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
+			text->bv_val, 0, 0 );
 		goto done;
 	}
 
-	rc = bdb_index_entry_add( &op, tid, e );
+	/* id2entry index */
+	rc = bdb_id2entry_add( be, tid, e );
 	if( rc != 0 ) {
 		snprintf( text->bv_val, text->bv_len,
-				"index_entry_add failed: %s (%d)",
+				"id2entry_add failed: %s (%d)",
 				db_strerror(rc), rc );
-#ifdef NEW_LOGGING
-		LDAP_LOG ( TOOLS, ERR, 
-			"=> bdb_tool_entry_put: %s\n", text->bv_val, 0, 0 );
-#else
 		Debug( LDAP_DEBUG_ANY,
-			"=> bdb_tool_entry_put: %s\n", text->bv_val, 0, 0 );
-#endif
+			"=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
+			text->bv_val, 0, 0 );
 		goto done;
 	}
 
 done:
 	if( rc == 0 ) {
+		if ( !( slapMode & SLAP_TOOL_QUICK )) {
 		rc = TXN_COMMIT( tid, 0 );
 		if( rc != 0 ) {
 			snprintf( text->bv_val, text->bv_len,
 					"txn_commit failed: %s (%d)",
 					db_strerror(rc), rc );
-#ifdef NEW_LOGGING
-			LDAP_LOG ( TOOLS, ERR, 
-				"=> bdb_tool_entry_put: %s\n", text->bv_val, 0, 0 );
-#else
 			Debug( LDAP_DEBUG_ANY,
-				"=> bdb_tool_entry_put: %s\n",
+				"=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
 				text->bv_val, 0, 0 );
-#endif
 			e->e_id = NOID;
 		}
+		}
 
 	} else {
+		if ( !( slapMode & SLAP_TOOL_QUICK )) {
 		TXN_ABORT( tid );
 		snprintf( text->bv_val, text->bv_len,
 			"txn_aborted! %s (%d)",
 			db_strerror(rc), rc );
-#ifdef NEW_LOGGING
-		LDAP_LOG ( TOOLS, ERR, 
-			"=> bdb_tool_entry_put: %s\n", text->bv_val, 0, 0 );
-#else
 		Debug( LDAP_DEBUG_ANY,
-			"=> bdb_tool_entry_put: %s\n",
+			"=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
 			text->bv_val, 0, 0 );
-#endif
+		}
 		e->e_id = NOID;
 	}
 
@@ -403,43 +547,45 @@ int bdb_tool_entry_reindex(
 	Entry *e;
 	DB_TXN *tid = NULL;
 	Operation op = {0};
+	Opheader ohdr = {0};
 
-#ifdef NEW_LOGGING
-	LDAP_LOG ( TOOLS, ARGS, 
-		"=> bdb_tool_entry_reindex( %ld )\n", (long) id, 0, 0 );
-#else
-	Debug( LDAP_DEBUG_ARGS, "=> bdb_tool_entry_reindex( %ld )\n",
+	Debug( LDAP_DEBUG_ARGS,
+		"=> " LDAP_XSTRING(bdb_tool_entry_reindex) "( %ld )\n",
 		(long) id, 0, 0 );
-#endif
+
+	/* No indexes configured, nothing to do. Could return an
+	 * error here to shortcut things.
+	 */
+	if (!bi->bi_attrs) {
+		return 0;
+	}
+
+	/* Get the first attribute to index */
+	if (bi->bi_linear_index && !index_nattrs) {
+		index_nattrs = bi->bi_nattrs - 1;
+		bi->bi_nattrs = 1;
+	}
 
 	e = bdb_tool_entry_get( be, id );
 
 	if( e == NULL ) {
-#ifdef NEW_LOGGING
-		LDAP_LOG ( TOOLS, DETAIL1, 
-			"bdb_tool_entry_reindex:: could not locate id=%ld\n", 
-			(long) id, 0, 0 );
-#else
 		Debug( LDAP_DEBUG_ANY,
-			"bdb_tool_entry_reindex:: could not locate id=%ld\n",
+			LDAP_XSTRING(bdb_tool_entry_reindex)
+			": could not locate id=%ld\n",
 			(long) id, 0, 0 );
-#endif
 		return -1;
 	}
 
+	if (! (slapMode & SLAP_TOOL_QUICK)) {
 	rc = TXN_BEGIN( bi->bi_dbenv, NULL, &tid, bi->bi_db_opflags );
 	if( rc != 0 ) {
-#ifdef NEW_LOGGING
-		LDAP_LOG ( TOOLS, ERR, 
-			"=> bdb_tool_entry_reindex: txn_begin failed: %s (%d)\n", 
-			db_strerror(rc), rc, 0 );
-#else
 		Debug( LDAP_DEBUG_ANY,
-			"=> bdb_tool_entry_reindex: txn_begin failed: %s (%d)\n",
+			"=> " LDAP_XSTRING(bdb_tool_entry_reindex) ": "
+			"txn_begin failed: %s (%d)\n",
 			db_strerror(rc), rc, 0 );
-#endif
 		goto done;
 	}
+	}
  	
 	/*
 	 * just (re)add them for now
@@ -448,64 +594,38 @@ int bdb_tool_entry_reindex(
 	 *
 	 */
 
-#ifdef NEW_LOGGING
-	LDAP_LOG ( TOOLS, ERR, 
-		"=> bdb_tool_entry_reindex( %ld, \"%s\" )\n", (long) id, e->e_dn, 0 );
-#else
-	Debug( LDAP_DEBUG_TRACE, "=> bdb_tool_entry_reindex( %ld, \"%s\" )\n",
+	Debug( LDAP_DEBUG_TRACE,
+		"=> " LDAP_XSTRING(bdb_tool_entry_reindex) "( %ld, \"%s\" )\n",
 		(long) id, e->e_dn, 0 );
-#endif
 
+	op.o_hdr = &ohdr;
 	op.o_bd = be;
 	op.o_tmpmemctx = NULL;
 	op.o_tmpmfuncs = &ch_mfuncs;
 
-#ifndef BDB_HIER
-	/* add dn2id indices */
-	rc = bdb_dn2id_add( &op, tid, NULL, e );
-	if( rc != 0 && rc != DB_KEYEXIST ) {
-#ifdef NEW_LOGGING
-		LDAP_LOG ( TOOLS, ERR, 
-			"=> bdb_tool_entry_reindex: dn2id_add failed: %s (%d)\n", 
-			db_strerror(rc), rc, 0 );
-#else
-		Debug( LDAP_DEBUG_ANY,
-			"=> bdb_tool_entry_reindex: dn2id_add failed: %s (%d)\n",
-			db_strerror(rc), rc, 0 );
-#endif
-		goto done;
-	}
-#endif
-
-	rc = bdb_index_entry_add( &op, tid, e );
+	rc = bdb_tool_index_add( &op, tid, e );
 
 done:
 	if( rc == 0 ) {
+		if (! (slapMode & SLAP_TOOL_QUICK)) {
 		rc = TXN_COMMIT( tid, 0 );
 		if( rc != 0 ) {
-#ifdef NEW_LOGGING
-			LDAP_LOG ( TOOLS, ERR, 
-				"=> bdb_tool_entry_reindex: txn_commit failed: %s (%d)\n", 
-				db_strerror(rc), rc, 0 );
-#else
 			Debug( LDAP_DEBUG_ANY,
-				"=> bdb_tool_entry_reindex: txn_commit failed: %s (%d)\n",
+				"=> " LDAP_XSTRING(bdb_tool_entry_reindex)
+				": txn_commit failed: %s (%d)\n",
 				db_strerror(rc), rc, 0 );
-#endif
 			e->e_id = NOID;
 		}
+		}
 
 	} else {
+		if (! (slapMode & SLAP_TOOL_QUICK)) {
 		TXN_ABORT( tid );
-#ifdef NEW_LOGGING
-		LDAP_LOG ( TOOLS, DETAIL1, 
-			"=> bdb_tool_entry_reindex: txn_aborted! %s (%d)\n", 
-			db_strerror(rc), rc, 0 );
-#else
 		Debug( LDAP_DEBUG_ANY,
-			"=> bdb_tool_entry_reindex: txn_aborted! %s (%d)\n",
+			"=> " LDAP_XSTRING(bdb_tool_entry_reindex)
+			": txn_aborted! %s (%d)\n",
 			db_strerror(rc), rc, 0 );
-#endif
+		}
 		e->e_id = NOID;
 	}
 	bdb_entry_release( &op, e, 0 );
@@ -522,41 +642,36 @@ ID bdb_tool_entry_modify(
 	struct bdb_info *bdb = (struct bdb_info *) be->be_private;
 	DB_TXN *tid = NULL;
 	Operation op = {0};
+	Opheader ohdr = {0};
 
 	assert( be != NULL );
 	assert( slapMode & SLAP_TOOL_MODE );
 
-	assert( text );
-	assert( text->bv_val );
+	assert( text != NULL );
+	assert( text->bv_val != NULL );
 	assert( text->bv_val[0] == '\0' );	/* overconservative? */
 
 	assert ( e->e_id != NOID );
-	assert ( e->e_id != 0 );
 
-#ifdef NEW_LOGGING
-	LDAP_LOG ( TOOLS, ARGS, "=> bdb_tool_entry_put( %ld, \"%s\" )\n",
-		(long) e->e_id, e->e_dn, 0 );
-#else
-	Debug( LDAP_DEBUG_TRACE, "=> bdb_tool_entry_put( %ld, \"%s\" )\n",
+	Debug( LDAP_DEBUG_TRACE,
+		"=> " LDAP_XSTRING(bdb_tool_entry_modify) "( %ld, \"%s\" )\n",
 		(long) e->e_id, e->e_dn, 0 );
-#endif
 
+	if (! (slapMode & SLAP_TOOL_QUICK)) {
 	rc = TXN_BEGIN( bdb->bi_dbenv, NULL, &tid, 
 		bdb->bi_db_opflags );
 	if( rc != 0 ) {
 		snprintf( text->bv_val, text->bv_len,
 			"txn_begin failed: %s (%d)",
 			db_strerror(rc), rc );
-#ifdef NEW_LOGGING
-	LDAP_LOG ( TOOLS, ERR, "=> bdb_tool_entry_put: %s\n", text->bv_val, 0, 0 );
-#else
 		Debug( LDAP_DEBUG_ANY,
-			"=> bdb_tool_entry_put: %s\n",
+			"=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n",
 			 text->bv_val, 0, 0 );
-#endif
 		return NOID;
 	}
+	}
 
+	op.o_hdr = &ohdr;
 	op.o_bd = be;
 	op.o_tmpmemctx = NULL;
 	op.o_tmpmfuncs = &ch_mfuncs;
@@ -567,79 +682,348 @@ ID bdb_tool_entry_modify(
 		snprintf( text->bv_val, text->bv_len,
 				"id2entry_add failed: %s (%d)",
 				db_strerror(rc), rc );
-#ifdef NEW_LOGGING
-		LDAP_LOG ( TOOLS, ERR, 
-			"=> bdb_tool_entry_put: %s\n", text->bv_val, 0, 0 );
-#else
-		Debug( LDAP_DEBUG_ANY,
-			"=> bdb_tool_entry_put: %s\n", text->bv_val, 0, 0 );
-#endif
-		goto done;
-	}
-
-	rc = bdb_index_entry_del( &op, tid, e );
-	if( rc != 0 ) {
-		snprintf( text->bv_val, text->bv_len,
-				"index_entry_del failed: %s (%d)",
-				db_strerror(rc), rc );
-#ifdef NEW_LOGGING
-		LDAP_LOG ( TOOLS, ERR, 
-			"=> bdb_tool_entry_put: %s\n", text->bv_val, 0, 0 );
-#else
 		Debug( LDAP_DEBUG_ANY,
-			"=> bdb_tool_entry_put: %s\n", text->bv_val, 0, 0 );
-#endif
-		goto done;
-	}
-
-	rc = bdb_index_entry_add( &op, tid, e );
-	if( rc != 0 ) {
-		snprintf( text->bv_val, text->bv_len,
-				"index_entry_add failed: %s (%d)",
-				db_strerror(rc), rc );
-#ifdef NEW_LOGGING
-		LDAP_LOG ( TOOLS, ERR, 
-			"=> bdb_tool_entry_put: %s\n", text->bv_val, 0, 0 );
-#else
-		Debug( LDAP_DEBUG_ANY,
-			"=> bdb_tool_entry_put: %s\n", text->bv_val, 0, 0 );
-#endif
+			"=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n",
+			text->bv_val, 0, 0 );
 		goto done;
 	}
 
 done:
 	if( rc == 0 ) {
+		if (! (slapMode & SLAP_TOOL_QUICK)) {
 		rc = TXN_COMMIT( tid, 0 );
 		if( rc != 0 ) {
 			snprintf( text->bv_val, text->bv_len,
 					"txn_commit failed: %s (%d)",
 					db_strerror(rc), rc );
-#ifdef NEW_LOGGING
-			LDAP_LOG ( TOOLS, ERR, 
-				"=> bdb_tool_entry_put: %s\n", text->bv_val, 0, 0 );
-#else
 			Debug( LDAP_DEBUG_ANY,
-				"=> bdb_tool_entry_put: %s\n",
-				text->bv_val, 0, 0 );
-#endif
+				"=> " LDAP_XSTRING(bdb_tool_entry_modify) ": "
+				"%s\n", text->bv_val, 0, 0 );
 			e->e_id = NOID;
 		}
+		}
 
 	} else {
+		if (! (slapMode & SLAP_TOOL_QUICK)) {
 		TXN_ABORT( tid );
 		snprintf( text->bv_val, text->bv_len,
 			"txn_aborted! %s (%d)",
 			db_strerror(rc), rc );
-#ifdef NEW_LOGGING
-		LDAP_LOG ( TOOLS, ERR, 
-			"=> bdb_tool_entry_put: %s\n", text->bv_val, 0, 0 );
-#else
 		Debug( LDAP_DEBUG_ANY,
-			"=> bdb_tool_entry_put: %s\n",
+			"=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n",
 			text->bv_val, 0, 0 );
-#endif
+		}
 		e->e_id = NOID;
 	}
 
 	return e->e_id;
 }
+
+#ifdef BDB_TOOL_IDL_CACHING
+static int
+bdb_tool_idl_cmp( const void *v1, const void *v2 )
+{
+	const bdb_tool_idl_cache *c1 = v1, *c2 = v2;
+	int rc;
+
+	if (( rc = c1->kstr.bv_len - c2->kstr.bv_len )) return rc;
+	return memcmp( c1->kstr.bv_val, c2->kstr.bv_val, c1->kstr.bv_len );
+}
+
+static int
+bdb_tool_idl_flush_one( void *v1, void *arg )
+{
+	bdb_tool_idl_cache *ic = v1;
+	DB *db = arg;
+	struct bdb_info *bdb = bdb_tool_info;
+	bdb_tool_idl_cache_entry *ice;
+	DBC *curs;
+	DBT key, data;
+	int i, rc;
+	ID id, nid;
+
+	/* Freshly allocated, ignore it */
+	if ( !ic->head && ic->count <= BDB_IDL_DB_SIZE ) {
+		return 0;
+	}
+
+	rc = db->cursor( db, NULL, &curs, 0 );
+	if ( rc )
+		return -1;
+
+	DBTzero( &key );
+	DBTzero( &data );
+
+	bv2DBT( &ic->kstr, &key );
+
+	data.size = data.ulen = sizeof( ID );
+	data.flags = DB_DBT_USERMEM;
+	data.data = &nid;
+
+	rc = curs->c_get( curs, &key, &data, DB_SET );
+	/* If key already exists and we're writing a range... */
+	if ( rc == 0 && ic->count > BDB_IDL_DB_SIZE ) {
+		/* If it's not currently a range, must delete old info */
+		if ( nid ) {
+			/* Skip lo */
+			while ( curs->c_get( curs, &key, &data, DB_NEXT_DUP ) == 0 )
+				curs->c_del( curs, 0 );
+
+			nid = 0;
+			/* Store range marker */
+			curs->c_put( curs, &key, &data, DB_KEYFIRST );
+		} else {
+			
+			/* Skip lo */
+			rc = curs->c_get( curs, &key, &data, DB_NEXT_DUP );
+
+			/* Get hi */
+			rc = curs->c_get( curs, &key, &data, DB_NEXT_DUP );
+
+			/* Delete hi */
+			curs->c_del( curs, 0 );
+		}
+		BDB_ID2DISK( ic->last, &nid );
+		curs->c_put( curs, &key, &data, DB_KEYLAST );
+		rc = 0;
+	} else if ( rc && rc != DB_NOTFOUND ) {
+		rc = -1;
+	} else if ( ic->count > BDB_IDL_DB_SIZE ) {
+		/* range, didn't exist before */
+		nid = 0;
+		rc = curs->c_put( curs, &key, &data, DB_KEYLAST );
+		if ( rc == 0 ) {
+			BDB_ID2DISK( ic->first, &nid );
+			rc = curs->c_put( curs, &key, &data, DB_KEYLAST );
+			if ( rc == 0 ) {
+				BDB_ID2DISK( ic->last, &nid );
+				rc = curs->c_put( curs, &key, &data, DB_KEYLAST );
+			}
+		}
+		if ( rc ) {
+			rc = -1;
+		}
+	} else {
+		int n;
+
+		/* Just a normal write */
+		rc = 0;
+		for ( ice = ic->head, n=0; ice; ice = ice->next, n++ ) {
+			int end;
+			if ( ice->next ) {
+				end = IDBLOCK;
+			} else {
+				end = ic->count & (IDBLOCK-1);
+				if ( !end )
+					end = IDBLOCK;
+			}
+			for ( i=0; i<end; i++ ) {
+				if ( !ice->ids[i] ) continue;
+				BDB_ID2DISK( ice->ids[i], &nid );
+				rc = curs->c_put( curs, &key, &data, DB_NODUPDATA );
+				if ( rc ) {
+					if ( rc == DB_KEYEXIST ) {
+						rc = 0;
+						continue;
+					}
+					rc = -1;
+					break;
+				}
+			}
+			if ( rc ) {
+				rc = -1;
+				break;
+			}
+		}
+		if ( ic->head ) {
+			ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
+			ic->tail->next = bdb_tool_idl_free_list;
+			bdb_tool_idl_free_list = ic->head;
+			bdb->bi_idl_cache_size -= n;
+			ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
+		}
+	}
+	if ( ic != db->app_private ) {
+		ch_free( ic );
+	} else {
+		ic->head = ic->tail = NULL;
+	}
+	curs->c_close( curs );
+	return rc;
+}
+
+static int
+bdb_tool_idl_flush_db( DB *db, bdb_tool_idl_cache *ic )
+{
+	Avlnode *root = db->app_private;
+	int rc;
+
+	db->app_private = ic;
+	rc = avl_apply( root, bdb_tool_idl_flush_one, db, -1, AVL_INORDER );
+	avl_free( root, NULL );
+	db->app_private = NULL;
+	if ( rc != -1 )
+		rc = 0;
+	return rc;
+}
+
+static int
+bdb_tool_idl_flush( BackendDB *be )
+{
+	struct bdb_info *bdb = (struct bdb_info *) be->be_private;
+	DB *db;
+	Avlnode *root;
+	int i, rc = 0;
+
+	for ( i=BDB_NDB; i < bdb->bi_ndatabases; i++ ) {
+		db = bdb->bi_databases[i]->bdi_db;
+		if ( !db->app_private ) continue;
+		rc = bdb_tool_idl_flush_db( db, NULL );
+		if ( rc )
+			break;
+	}
+	if ( !rc ) {
+		bdb->bi_idl_cache_size = 0;
+	}
+	return rc;
+}
+
+int bdb_tool_idl_add(
+	BackendDB *be,
+	DB *db,
+	DB_TXN *txn,
+	DBT *key,
+	ID id )
+{
+	struct bdb_info *bdb = (struct bdb_info *) be->be_private;
+	bdb_tool_idl_cache *ic, itmp;
+	bdb_tool_idl_cache_entry *ice;
+	int rc;
+
+	if ( !bdb->bi_idl_cache_max_size )
+		return bdb_idl_insert_key( be, db, txn, key, id );
+
+	DBT2bv( key, &itmp.kstr );
+
+	ic = avl_find( (Avlnode *)db->app_private, &itmp, bdb_tool_idl_cmp );
+
+	/* No entry yet, create one */
+	if ( !ic ) {
+		DBC *curs;
+		DBT data;
+		ID nid;
+		int rc;
+
+		ic = ch_malloc( sizeof( bdb_tool_idl_cache ) + itmp.kstr.bv_len );
+		ic->kstr.bv_len = itmp.kstr.bv_len;
+		ic->kstr.bv_val = (char *)(ic+1);
+		AC_MEMCPY( ic->kstr.bv_val, itmp.kstr.bv_val, ic->kstr.bv_len );
+		ic->head = ic->tail = NULL;
+		ic->last = 0;
+		ic->count = 0;
+		avl_insert( (Avlnode **)&db->app_private, ic, bdb_tool_idl_cmp,
+			avl_dup_error );
+
+		/* load existing key count here */
+		rc = db->cursor( db, NULL, &curs, 0 );
+		if ( rc ) return rc;
+
+		data.ulen = sizeof( ID );
+		data.flags = DB_DBT_USERMEM;
+		data.data = &nid;
+		rc = curs->c_get( curs, key, &data, DB_SET );
+		if ( rc == 0 ) {
+			if ( nid == 0 ) {
+				ic->count = BDB_IDL_DB_SIZE+1;
+			} else {
+				db_recno_t count;
+
+				curs->c_count( curs, &count, 0 );
+				ic->count = count;
+				BDB_DISK2ID( &nid, &ic->first );
+			}
+		}
+		curs->c_close( curs );
+	}
+	/* are we a range already? */
+	if ( ic->count > BDB_IDL_DB_SIZE ) {
+		ic->last = id;
+		return 0;
+	/* Are we at the limit, and converting to a range? */
+	} else if ( ic->count == BDB_IDL_DB_SIZE ) {
+		int n;
+		for ( ice = ic->head, n=0; ice; ice = ice->next, n++ )
+			/* counting */ ;
+		if ( n ) {
+			ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
+			ic->tail->next = bdb_tool_idl_free_list;
+			bdb_tool_idl_free_list = ic->head;
+			bdb->bi_idl_cache_size -= n;
+			ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
+		}
+		ic->head = ic->tail = NULL;
+		ic->last = id;
+		ic->count++;
+		return 0;
+	}
+	/* No free block, create that too */
+	if ( !ic->tail || ( ic->count & (IDBLOCK-1)) == 0) {
+		ice = NULL;
+		ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
+		if ( bdb->bi_idl_cache_size >= bdb->bi_idl_cache_max_size ) {
+			ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
+			rc = bdb_tool_idl_flush_db( db, ic );
+			if ( rc )
+				return rc;
+			avl_insert( (Avlnode **)&db->app_private, ic, bdb_tool_idl_cmp,
+				avl_dup_error );
+			ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
+		}
+		bdb->bi_idl_cache_size++;
+		if ( bdb_tool_idl_free_list ) {
+			ice = bdb_tool_idl_free_list;
+			bdb_tool_idl_free_list = ice->next;
+		}
+		ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
+		if ( !ice ) {
+			ice = ch_malloc( sizeof( bdb_tool_idl_cache_entry ));
+		}
+		memset( ice, 0, sizeof( *ice ));
+		if ( !ic->head ) {
+			ic->head = ice;
+		} else {
+			ic->tail->next = ice;
+		}
+		ic->tail = ice;
+		if ( !ic->count )
+			ic->first = id;
+	}
+	ice = ic->tail;
+	ice->ids[ ic->count & (IDBLOCK-1) ] = id;
+	ic->count++;
+
+	return 0;
+}
+#endif
+
+static void *
+bdb_tool_index_task( void *ctx, void *ptr )
+{
+	int base = *(int *)ptr;
+
+	free( ptr );
+	while ( 1 ) {
+		ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex );
+		bdb_tool_index_tcount--;
+		ldap_pvt_thread_cond_wait( &bdb_tool_index_cond,
+			&bdb_tool_index_mutex );
+		ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex );
+		if ( slapd_shutdown )
+			break;
+
+		bdb_tool_index_threads[base] = bdb_index_recrun( bdb_tool_ix_op,
+			bdb_tool_info, bdb_tool_index_rec, bdb_tool_ix_id, base );
+	}
+
+	return NULL;
+}