X-Git-Url: https://git.sur5r.net/?a=blobdiff_plain;f=servers%2Fslapd%2Fback-bdb%2Ftools.c;h=167f3c7e5e2b46d198a5e2114a5cba5933a87cf3;hb=0af1940f3fb59fe57b2281ef253fe1341c505c2c;hp=25cbd8b55f26000092f67ad12696b46a13239a1c;hpb=34e154e28b8480466704cfdc357c18614090717d;p=openldap

diff --git a/servers/slapd/back-bdb/tools.c b/servers/slapd/back-bdb/tools.c
index 25cbd8b55f..167f3c7e5e 100644
--- a/servers/slapd/back-bdb/tools.c
+++ b/servers/slapd/back-bdb/tools.c
@@ -2,7 +2,7 @@
 /* $OpenLDAP$ */
 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
  *
- * Copyright 2000-2005 The OpenLDAP Foundation.
+ * Copyright 2000-2006 The OpenLDAP Foundation.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -36,14 +36,42 @@ static dn_id hbuf[HOLE_SIZE], *holes = hbuf;
 static unsigned nhmax = HOLE_SIZE;
 static unsigned nholes;
 
-static Avlnode *index_attrs, index_dummy;
+static int index_nattrs;
 
+#ifdef BDB_TOOL_IDL_CACHING
 #define bdb_tool_idl_cmp		BDB_SYMBOL(tool_idl_cmp)
 #define bdb_tool_idl_flush_one		BDB_SYMBOL(tool_idl_flush_one)
 #define bdb_tool_idl_flush		BDB_SYMBOL(tool_idl_flush)
 
 static int bdb_tool_idl_flush( BackendDB *be );
 
+#define	IDBLOCK	1024
+
+typedef struct bdb_tool_idl_cache_entry {
+	struct bdb_tool_idl_cache_entry *next;
+	ID ids[IDBLOCK];
+} bdb_tool_idl_cache_entry;
+ 
+typedef struct bdb_tool_idl_cache {
+	struct berval kstr;
+	bdb_tool_idl_cache_entry *head, *tail;
+	ID first, last;
+	int count;
+} bdb_tool_idl_cache;
+
+static bdb_tool_idl_cache_entry *bdb_tool_idl_free_list;
+#endif	/* BDB_TOOL_IDL_CACHING */
+
+static ID bdb_tool_ix_id;
+static Operation *bdb_tool_ix_op;
+static int *bdb_tool_index_threads, bdb_tool_index_tcount;
+static void *bdb_tool_index_rec;
+static struct bdb_info *bdb_tool_info;
+static ldap_pvt_thread_mutex_t bdb_tool_index_mutex;
+static ldap_pvt_thread_cond_t bdb_tool_index_cond;
+
+static void * bdb_tool_index_task( void *ctx, void *ptr );
+
 int bdb_tool_entry_open(
 	BackendDB *be, int mode )
 {
@@ -64,13 +92,39 @@ int bdb_tool_entry_open(
 		}
 	}
 
+	/* Set up for threaded slapindex */
+	if (( slapMode & (SLAP_TOOL_QUICK|SLAP_TOOL_READONLY)) == SLAP_TOOL_QUICK
+		&& bdb->bi_nattrs ) {
+		if ( !bdb_tool_info ) {
+			int i;
+			ldap_pvt_thread_mutex_init( &bdb_tool_index_mutex );
+			ldap_pvt_thread_cond_init( &bdb_tool_index_cond );
+			bdb_tool_index_threads = ch_malloc( slap_tool_thread_max * sizeof( int ));
+			bdb_tool_index_rec = ch_malloc( bdb->bi_nattrs * sizeof( IndexRec ));
+			bdb_tool_index_tcount = slap_tool_thread_max - 1;
+			for (i=1; i<slap_tool_thread_max; i++) {
+				int *ptr = ch_malloc( sizeof( int ));
+				*ptr = i;
+				ldap_pvt_thread_pool_submit( &connection_pool,
+					bdb_tool_index_task, ptr );
+			}
+		}
+		bdb_tool_info = bdb;
+	}
+
 	return 0;
 }
 
 int bdb_tool_entry_close(
 	BackendDB *be )
 {
-	assert( be != NULL );
+	if ( bdb_tool_info ) {
+		slapd_shutdown = 1;
+		ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex );
+		bdb_tool_index_tcount = slap_tool_thread_max - 1;
+		ldap_pvt_thread_cond_broadcast( &bdb_tool_index_cond );
+		ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex );
+	}
 
 	if( key.data ) {
 		ch_free( key.data );
@@ -86,7 +140,9 @@ int bdb_tool_entry_close(
 		cursor = NULL;
 	}
 
+#ifdef BDB_TOOL_IDL_CACHING
 	bdb_tool_idl_flush( be );
+#endif
 
 	if( nholes ) {
 		unsigned i;
@@ -101,8 +157,6 @@ int bdb_tool_entry_close(
 	return 0;
 }
 
-static int bdb_reindex_cmp(const void *a, const void *b) { return 0; }
-
 ID bdb_tool_entry_next(
 	BackendDB *be )
 {
@@ -120,14 +174,13 @@ ID bdb_tool_entry_next(
 		/* If we're doing linear indexing and there are more attrs to
 		 * index, and we're at the end of the database, start over.
 		 */
-		if ( bdb->bi_attrs == &index_dummy ) {
-			if ( index_attrs && rc == DB_NOTFOUND ) {
-				/* optional - do a checkpoint here? */
-				index_dummy.avl_data = avl_delete(&index_attrs, NULL, bdb_reindex_cmp);
-				rc = cursor->c_get( cursor, &key, &data, DB_FIRST );
-			}
+		if ( index_nattrs && rc == DB_NOTFOUND ) {
+			/* optional - do a checkpoint here? */
+			bdb_attr_info_free( bdb->bi_attrs[0] );
+			bdb->bi_attrs[0] = bdb->bi_attrs[index_nattrs];
+			index_nattrs--;
+			rc = cursor->c_get( cursor, &key, &data, DB_FIRST );
 			if ( rc ) {
-				bdb->bi_attrs = NULL;
 				return NOID;
 			}
 		} else {
@@ -214,7 +267,7 @@ Entry* bdb_tool_entry_get( BackendDB *be, ID id )
 		e->e_id = id;
 	}
 #ifdef BDB_HIER
-	{
+	if ( slapMode & SLAP_TOOL_READONLY ) {
 		EntryInfo *ei = NULL;
 		Operation op = {0};
 		Opheader ohdr = {0};
@@ -330,6 +383,60 @@ static int bdb_tool_next_id(
 	return rc;
 }
 
+static int
+bdb_tool_index_add(
+	Operation *op,
+	DB_TXN *txn,
+	Entry *e )
+{
+	struct bdb_info *bdb = (struct bdb_info *) op->o_bd->be_private;
+
+	if ( slapMode & SLAP_TOOL_QUICK ) {
+		IndexRec *ir;
+		int i, rc;
+		Attribute *a;
+		
+		ir = bdb_tool_index_rec;
+		memset(ir, 0, bdb->bi_nattrs * sizeof( IndexRec ));
+
+		for ( a = e->e_attrs; a != NULL; a = a->a_next ) {
+			rc = bdb_index_recset( bdb, a, a->a_desc->ad_type, 
+				&a->a_desc->ad_tags, ir );
+			if ( rc )
+				return rc;
+		}
+		bdb_tool_ix_id = e->e_id;
+		bdb_tool_ix_op = op;
+		ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex );
+		/* Wait for all threads to be ready */
+		while ( bdb_tool_index_tcount ) {
+			ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex );
+			ldap_pvt_thread_yield();
+			ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex );
+		}
+		for ( i=1; i<slap_tool_thread_max; i++ )
+			bdb_tool_index_threads[i] = LDAP_BUSY;
+		bdb_tool_index_tcount = slap_tool_thread_max - 1;
+		ldap_pvt_thread_cond_broadcast( &bdb_tool_index_cond );
+		ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex );
+		rc = bdb_index_recrun( op, bdb, ir, e->e_id, 0 );
+		if ( rc )
+			return rc;
+		for ( i=1; i<slap_tool_thread_max; i++ ) {
+			if ( bdb_tool_index_threads[i] == LDAP_BUSY ) {
+				ldap_pvt_thread_yield();
+				i--;
+				continue;
+			}
+			if ( bdb_tool_index_threads[i] )
+				return bdb_tool_index_threads[i];
+		}
+		return 0;
+	} else {
+		return bdb_index_entry_add( op, txn, e );
+	}
+}
+
 ID bdb_tool_entry_put(
 	BackendDB *be,
 	Entry *e,
@@ -376,11 +483,11 @@ ID bdb_tool_entry_put(
 		goto done;
 	}
 
-	/* id2entry index */
-	rc = bdb_id2entry_add( be, tid, e );
+	if ( !bdb->bi_linear_index )
+		rc = bdb_tool_index_add( &op, tid, e );
 	if( rc != 0 ) {
 		snprintf( text->bv_val, text->bv_len,
-				"id2entry_add failed: %s (%d)",
+				"index_entry_add failed: %s (%d)",
 				db_strerror(rc), rc );
 		Debug( LDAP_DEBUG_ANY,
 			"=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
@@ -388,11 +495,11 @@ ID bdb_tool_entry_put(
 		goto done;
 	}
 
-	if ( !bdb->bi_linear_index )
-		rc = bdb_index_entry_add( &op, tid, e );
+	/* id2entry index */
+	rc = bdb_id2entry_add( be, tid, e );
 	if( rc != 0 ) {
 		snprintf( text->bv_val, text->bv_len,
-				"index_entry_add failed: %s (%d)",
+				"id2entry_add failed: %s (%d)",
 				db_strerror(rc), rc );
 		Debug( LDAP_DEBUG_ANY,
 			"=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
@@ -454,10 +561,9 @@ int bdb_tool_entry_reindex(
 	}
 
 	/* Get the first attribute to index */
-	if (bi->bi_linear_index && !index_attrs && bi->bi_attrs != &index_dummy) {
-		index_attrs = bi->bi_attrs;
-		bi->bi_attrs = &index_dummy;
-		index_dummy.avl_data = avl_delete(&index_attrs, NULL, bdb_reindex_cmp);
+	if (bi->bi_linear_index && !index_nattrs) {
+		index_nattrs = bi->bi_nattrs - 1;
+		bi->bi_nattrs = 1;
 	}
 
 	e = bdb_tool_entry_get( be, id );
@@ -497,7 +603,7 @@ int bdb_tool_entry_reindex(
 	op.o_tmpmemctx = NULL;
 	op.o_tmpmfuncs = &ch_mfuncs;
 
-	rc = bdb_index_entry_add( &op, tid, e );
+	rc = bdb_tool_index_add( &op, tid, e );
 
 done:
 	if( rc == 0 ) {
@@ -582,33 +688,6 @@ ID bdb_tool_entry_modify(
 		goto done;
 	}
 
-#if 0
-	/* FIXME: this is bogus, we don't have the old values to delete
-	 * from the index because the given entry has already been modified.
-	 */
-	rc = bdb_index_entry_del( &op, tid, e );
-	if( rc != 0 ) {
-		snprintf( text->bv_val, text->bv_len,
-				"index_entry_del failed: %s (%d)",
-				db_strerror(rc), rc );
-		Debug( LDAP_DEBUG_ANY,
-			"=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n",
-			text->bv_val, 0, 0 );
-		goto done;
-	}
-#endif
-
-	rc = bdb_index_entry_add( &op, tid, e );
-	if( rc != 0 ) {
-		snprintf( text->bv_val, text->bv_len,
-				"index_entry_add failed: %s (%d)",
-				db_strerror(rc), rc );
-		Debug( LDAP_DEBUG_ANY,
-			"=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n",
-			text->bv_val, 0, 0 );
-		goto done;
-	}
-
 done:
 	if( rc == 0 ) {
 		if (! (slapMode & SLAP_TOOL_QUICK)) {
@@ -640,22 +719,7 @@ done:
 	return e->e_id;
 }
 
-#define	IDBLOCK	1024
-
-typedef struct bdb_tool_idl_cache_entry {
-	struct bdb_tool_idl_cache_entry *next;
-	ID ids[IDBLOCK];
-} bdb_tool_idl_cache_entry;
-
-typedef struct bdb_tool_idl_cache {
-	struct berval kstr;
-	bdb_tool_idl_cache_entry *head, *tail;
-	ID first, last;
-	int count;
-} bdb_tool_idl_cache;
-
-static bdb_tool_idl_cache_entry *bdb_tool_idl_free_list;
-
+#ifdef BDB_TOOL_IDL_CACHING
 static int
 bdb_tool_idl_cmp( const void *v1, const void *v2 )
 {
@@ -671,12 +735,18 @@ bdb_tool_idl_flush_one( void *v1, void *arg )
 {
 	bdb_tool_idl_cache *ic = v1;
 	DB *db = arg;
+	struct bdb_info *bdb = bdb_tool_info;
 	bdb_tool_idl_cache_entry *ice;
 	DBC *curs;
 	DBT key, data;
 	int i, rc;
 	ID id, nid;
 
+	/* Freshly allocated, ignore it */
+	if ( !ic->head && ic->count <= BDB_IDL_DB_SIZE ) {
+		return 0;
+	}
+
 	rc = db->cursor( db, NULL, &curs, 0 );
 	if ( rc )
 		return -1;
@@ -692,7 +762,7 @@ bdb_tool_idl_flush_one( void *v1, void *arg )
 
 	rc = curs->c_get( curs, &key, &data, DB_SET );
 	/* If key already exists and we're writing a range... */
-	if ( rc == 0 && ic->head == NULL ) {
+	if ( rc == 0 && ic->count > BDB_IDL_DB_SIZE ) {
 		/* If it's not currently a range, must delete old info */
 		if ( nid ) {
 			/* Skip lo */
@@ -718,7 +788,7 @@ bdb_tool_idl_flush_one( void *v1, void *arg )
 		rc = 0;
 	} else if ( rc && rc != DB_NOTFOUND ) {
 		rc = -1;
-	} else if ( ic->head == NULL ) {
+	} else if ( ic->count > BDB_IDL_DB_SIZE ) {
 		/* range, didn't exist before */
 		nid = 0;
 		rc = curs->c_put( curs, &key, &data, DB_KEYLAST );
@@ -734,11 +804,19 @@ bdb_tool_idl_flush_one( void *v1, void *arg )
 			rc = -1;
 		}
 	} else {
+		int n;
+
 		/* Just a normal write */
 		rc = 0;
-		for ( ice = ic->head; ice; ice = ic->head ) {
-			int end = ice->next ? IDBLOCK : (ic->count & (IDBLOCK-1));
-			ic->head = ice->next;
+		for ( ice = ic->head, n=0; ice; ice = ice->next, n++ ) {
+			int end;
+			if ( ice->next ) {
+				end = IDBLOCK;
+			} else {
+				end = ic->count & (IDBLOCK-1);
+				if ( !end )
+					end = IDBLOCK;
+			}
 			for ( i=0; i<end; i++ ) {
 				if ( !ice->ids[i] ) continue;
 				BDB_ID2DISK( ice->ids[i], &nid );
@@ -752,19 +830,43 @@ bdb_tool_idl_flush_one( void *v1, void *arg )
 					break;
 				}
 			}
-			ice->next = bdb_tool_idl_free_list;
-			bdb_tool_idl_free_list = ice;
 			if ( rc ) {
 				rc = -1;
 				break;
 			}
 		}
+		if ( ic->head ) {
+			ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
+			ic->tail->next = bdb_tool_idl_free_list;
+			bdb_tool_idl_free_list = ic->head;
+			bdb->bi_idl_cache_size -= n;
+			ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
+		}
+	}
+	if ( ic != db->app_private ) {
+		ch_free( ic );
+	} else {
+		ic->head = ic->tail = NULL;
 	}
-	ch_free( ic );
 	curs->c_close( curs );
 	return rc;
 }
 
+static int
+bdb_tool_idl_flush_db( DB *db, bdb_tool_idl_cache *ic )
+{
+	Avlnode *root = db->app_private;
+	int rc;
+
+	db->app_private = ic;
+	rc = avl_apply( root, bdb_tool_idl_flush_one, db, -1, AVL_INORDER );
+	avl_free( root, NULL );
+	db->app_private = NULL;
+	if ( rc != -1 )
+		rc = 0;
+	return rc;
+}
+
 static int
 bdb_tool_idl_flush( BackendDB *be )
 {
@@ -776,15 +878,13 @@ bdb_tool_idl_flush( BackendDB *be )
 	for ( i=BDB_NDB; i < bdb->bi_ndatabases; i++ ) {
 		db = bdb->bi_databases[i]->bdi_db;
 		if ( !db->app_private ) continue;
-		rc = avl_apply( db->app_private, bdb_tool_idl_flush_one, db, -1,
-			AVL_INORDER );
-		avl_free( db->app_private, NULL );
-		db->app_private = NULL;
-		if ( rc == -1 ) break;
-		rc = 0;
+		rc = bdb_tool_idl_flush_db( db, NULL );
+		if ( rc )
+			break;
 	}
-	if ( !rc )
+	if ( !rc ) {
 		bdb->bi_idl_cache_size = 0;
+	}
 	return rc;
 }
 
@@ -798,17 +898,14 @@ int bdb_tool_idl_add(
 	struct bdb_info *bdb = (struct bdb_info *) be->be_private;
 	bdb_tool_idl_cache *ic, itmp;
 	bdb_tool_idl_cache_entry *ice;
-	Avlnode *root;
 	int rc;
 
 	if ( !bdb->bi_idl_cache_max_size )
 		return bdb_idl_insert_key( be, db, txn, key, id );
 
-	root = db->app_private;
-
 	DBT2bv( key, &itmp.kstr );
 
-	ic = avl_find( root, &itmp, bdb_tool_idl_cmp );
+	ic = avl_find( (Avlnode *)db->app_private, &itmp, bdb_tool_idl_cmp );
 
 	/* No entry yet, create one */
 	if ( !ic ) {
@@ -824,8 +921,8 @@ int bdb_tool_idl_add(
 		ic->head = ic->tail = NULL;
 		ic->last = 0;
 		ic->count = 0;
-		avl_insert( &root, ic, bdb_tool_idl_cmp, avl_dup_error );
-		db->app_private = root;
+		avl_insert( (Avlnode **)&db->app_private, ic, bdb_tool_idl_cmp,
+			avl_dup_error );
 
 		/* load existing key count here */
 		rc = db->cursor( db, NULL, &curs, 0 );
@@ -854,11 +951,16 @@ int bdb_tool_idl_add(
 		return 0;
 	/* Are we at the limit, and converting to a range? */
 	} else if ( ic->count == BDB_IDL_DB_SIZE ) {
-		for (ice = ic->head; ice; ice = ice->next ) {
-			bdb->bi_idl_cache_size--;
+		int n;
+		for ( ice = ic->head, n=0; ice; ice = ice->next, n++ )
+			/* counting */ ;
+		if ( n ) {
+			ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
+			ic->tail->next = bdb_tool_idl_free_list;
+			bdb_tool_idl_free_list = ic->head;
+			bdb->bi_idl_cache_size -= n;
+			ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
 		}
-		ic->tail->next = bdb_tool_idl_free_list;
-		bdb_tool_idl_free_list = ic->head;
 		ic->head = ic->tail = NULL;
 		ic->last = id;
 		ic->count++;
@@ -866,10 +968,24 @@ int bdb_tool_idl_add(
 	}
 	/* No free block, create that too */
 	if ( !ic->tail || ( ic->count & (IDBLOCK-1)) == 0) {
+		ice = NULL;
+		ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
+		if ( bdb->bi_idl_cache_size >= bdb->bi_idl_cache_max_size ) {
+			ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
+			rc = bdb_tool_idl_flush_db( db, ic );
+			if ( rc )
+				return rc;
+			avl_insert( (Avlnode **)&db->app_private, ic, bdb_tool_idl_cmp,
+				avl_dup_error );
+			ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
+		}
+		bdb->bi_idl_cache_size++;
 		if ( bdb_tool_idl_free_list ) {
 			ice = bdb_tool_idl_free_list;
 			bdb_tool_idl_free_list = ice->next;
-		} else {
+		}
+		ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
+		if ( !ice ) {
 			ice = ch_malloc( sizeof( bdb_tool_idl_cache_entry ));
 		}
 		memset( ice, 0, sizeof( *ice ));
@@ -879,7 +995,6 @@ int bdb_tool_idl_add(
 			ic->tail->next = ice;
 		}
 		ic->tail = ice;
-		bdb->bi_idl_cache_size++;
 		if ( !ic->count )
 			ic->first = id;
 	}
@@ -887,10 +1002,28 @@ int bdb_tool_idl_add(
 	ice->ids[ ic->count & (IDBLOCK-1) ] = id;
 	ic->count++;
 
-	rc = 0;
-	/* Check for flush */
-	if ( bdb->bi_idl_cache_size > bdb->bi_idl_cache_max_size ) {
-		rc = bdb_tool_idl_flush( be );
+	return 0;
+}
+#endif
+
+static void *
+bdb_tool_index_task( void *ctx, void *ptr )
+{
+	int base = *(int *)ptr;
+
+	free( ptr );
+	while ( 1 ) {
+		ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex );
+		bdb_tool_index_tcount--;
+		ldap_pvt_thread_cond_wait( &bdb_tool_index_cond,
+			&bdb_tool_index_mutex );
+		ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex );
+		if ( slapd_shutdown )
+			break;
+
+		bdb_tool_index_threads[base] = bdb_index_recrun( bdb_tool_ix_op,
+			bdb_tool_info, bdb_tool_index_rec, bdb_tool_ix_id, base );
 	}
-	return rc;
+
+	return NULL;
 }