From dd7003cf9f1b5f48a915a99fde5962af73871c56 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Thu, 22 Jul 2004 22:32:37 +0000 Subject: [PATCH] Added linearindex option for slapindex. Improves performance when database size exceeds dbcache size by minimizing cache thrashing. --- doc/man/man5/slapd-bdb.5 | 8 ++++++ servers/slapd/back-bdb/back-bdb.h | 1 + servers/slapd/back-bdb/config.c | 4 +++ servers/slapd/back-bdb/tools.c | 42 ++++++++++++++++++++++++++++--- 4 files changed, 52 insertions(+), 3 deletions(-) diff --git a/doc/man/man5/slapd-bdb.5 b/doc/man/man5/slapd-bdb.5 index 05941827df..3798338d65 100644 --- a/doc/man/man5/slapd-bdb.5 +++ b/doc/man/man5/slapd-bdb.5 @@ -109,6 +109,14 @@ may be specified to disallow use of this index by named subtypes. Note: changing index settings requires rebuilding indices, see .BR slapindex (8). .TP +.B linearindex +Tell slapindex to index one attribute at a time. By default, all indexed +attributes in an entry are processed at the same time. With this option, +each indexed attribute is processed individually, using multiple passes +through the entire database. This option improves slapindex performance +when the database size exceeds the dbcache size. When the dbcache is +large enough, this option is not needed and will decrease performance. +.TP .B lockdetect {oldest|youngest|fewest|random|default} Specify which transaction to abort when a deadlock is detected. The default is the same as diff --git a/servers/slapd/back-bdb/back-bdb.h b/servers/slapd/back-bdb/back-bdb.h index 3dcbc6de96..fc286726ed 100644 --- a/servers/slapd/back-bdb/back-bdb.h +++ b/servers/slapd/back-bdb/back-bdb.h @@ -163,6 +163,7 @@ struct bdb_info { Avlnode *bi_attrs; void *bi_search_stack; int bi_search_stack_depth; + int bi_linear_index; int bi_txn_cp; u_int32_t bi_txn_cp_min; diff --git a/servers/slapd/back-bdb/config.c b/servers/slapd/back-bdb/config.c index d7eeba5e74..f48e065d40 100644 --- a/servers/slapd/back-bdb/config.c +++ b/servers/slapd/back-bdb/config.c @@ -69,6 +69,10 @@ bdb_db_config( slapMode & SLAP_TOOL_MODE ) { bdb->bi_dbenv_xflags |= DB_TXN_NOT_DURABLE; + /* slapindex algorithm tuning */ + } else if ( strcasecmp( argv[0], "linearindex" ) == 0 ) { + bdb->bi_linear_index = 1; + /* transaction checkpoint configuration */ } else if ( strcasecmp( argv[0], "checkpoint" ) == 0 ) { if ( argc < 3 ) { diff --git a/servers/slapd/back-bdb/tools.c b/servers/slapd/back-bdb/tools.c index 6484e4db9e..8898017156 100644 --- a/servers/slapd/back-bdb/tools.c +++ b/servers/slapd/back-bdb/tools.c @@ -19,6 +19,7 @@ #include #include +#define AVL_INTERNAL #include "back-bdb.h" #include "external.h" @@ -35,6 +36,8 @@ static dn_id hbuf[HOLE_SIZE], *holes = hbuf; static unsigned nhmax = HOLE_SIZE; static unsigned nholes; +Avlnode *index_attrs, index_dummy; + int bdb_tool_entry_open( BackendDB *be, int mode ) { @@ -79,6 +82,8 @@ int bdb_tool_entry_close( return 0; } +static int bdb_reindex_cmp(const void *a, const void *b) { return 0; } + ID bdb_tool_entry_next( BackendDB *be ) { @@ -90,6 +95,7 @@ ID bdb_tool_entry_next( assert( slapMode & SLAP_TOOL_MODE ); assert( bdb != NULL ); + /* Initialization */ if (cursor == NULL) { rc = bdb->bi_id2entry->bdi_db->cursor( bdb->bi_id2entry->bdi_db, NULL, &cursor, @@ -102,7 +108,22 @@ ID bdb_tool_entry_next( rc = cursor->c_get( cursor, &key, &data, DB_NEXT ); if( rc != 0 ) { - return NOID; + /* If we're doing linear indexing and there are more attrs to + * index, and we're at the end of the database, start over. + */ + if ( bdb->bi_attrs == &index_dummy ) { + if ( index_attrs && rc == DB_NOTFOUND ) { + /* optional - do a checkpoint here? */ + index_dummy.avl_data = avl_delete(&index_attrs, NULL, bdb_reindex_cmp); + rc = cursor->c_get( cursor, &key, &data, DB_FIRST ); + } + if ( rc ) { + bdb->bi_attrs = NULL; + return NOID; + } + } else { + return NOID; + } } if( data.data == NULL ) { @@ -354,7 +375,8 @@ ID bdb_tool_entry_put( goto done; } - rc = bdb_index_entry_add( &op, tid, e ); + if ( !bdb->bi_linear_index ) + rc = bdb_index_entry_add( &op, tid, e ); if( rc != 0 ) { snprintf( text->bv_val, text->bv_len, "index_entry_add failed: %s (%d)", @@ -424,6 +446,20 @@ int bdb_tool_entry_reindex( (long) id, 0, 0 ); #endif + /* No indexes configured, nothing to do. Could return an + * error here to shortcut things. + */ + if (!bi->bi_attrs) { + return 0; + } + + /* Get the first attribute to index */ + if (bi->bi_linear_index && !index_attrs && bi->bi_attrs != &index_dummy) { + index_attrs = bi->bi_attrs; + bi->bi_attrs = &index_dummy; + index_dummy.avl_data = avl_delete(&index_attrs, NULL, bdb_reindex_cmp); + } + e = bdb_tool_entry_get( be, id ); if( e == NULL ) { @@ -472,7 +508,7 @@ int bdb_tool_entry_reindex( op.o_tmpmemctx = NULL; op.o_tmpmfuncs = &ch_mfuncs; -#ifndef BDB_HIER +#if 0 /* ndef BDB_HIER */ /* add dn2id indices */ rc = bdb_dn2id_add( &op, tid, NULL, e ); if( rc != 0 && rc != DB_KEYEXIST ) { -- 2.39.5