Added linearindex option for slapindex. Improves performance when database

size exceeds dbcache size by minimizing cache thrashing.

diff --git a/doc/man/man5/slapd-bdb.5 b/doc/man/man5/slapd-bdb.5
index 05941827df451136de3c0c8e06962057ed14bc7d..3798338d65478c37d889418771354e9d032b1aa0 100644 (file)
--- a/doc/man/man5/slapd-bdb.5
+++ b/doc/man/man5/slapd-bdb.5
@@ -109,6 +109,14 @@ may be specified to disallow use of this index by named subtypes.
 Note: changing index settings requires rebuilding indices, see
 .BR slapindex (8).
 .TP
+.B linearindex
+Tell slapindex to index one attribute at a time. By default, all indexed
+attributes in an entry are processed at the same time. With this option,
+each indexed attribute is processed individually, using multiple passes
+through the entire database. This option improves slapindex performance
+when the database size exceeds the dbcache size. When the dbcache is
+large enough, this option is not needed and will decrease performance.
+.TP
 .B lockdetect {oldest|youngest|fewest|random|default}
 Specify which transaction to abort when a deadlock is detected.
 The default is the same as

diff --git a/servers/slapd/back-bdb/back-bdb.h b/servers/slapd/back-bdb/back-bdb.h
index 3dcbc6de96f266f18ebfebb9183a390352419901..fc286726edf922004790df1d53c23f802f370da7 100644 (file)
--- a/servers/slapd/back-bdb/back-bdb.h
+++ b/servers/slapd/back-bdb/back-bdb.h
@@ -163,6 +163,7 @@ struct bdb_info {
        Avlnode         *bi_attrs;
        void            *bi_search_stack;
        int             bi_search_stack_depth;
+       int             bi_linear_index;
 
        int                     bi_txn_cp;
        u_int32_t       bi_txn_cp_min;

diff --git a/servers/slapd/back-bdb/config.c b/servers/slapd/back-bdb/config.c
index d7eeba5e74bc6b3bdfbe22ba76a222f283d2b348..f48e065d4063d61c6a478c5f333d84d5feb137e3 100644 (file)
--- a/servers/slapd/back-bdb/config.c
+++ b/servers/slapd/back-bdb/config.c
@@ -69,6 +69,10 @@ bdb_db_config(
                slapMode & SLAP_TOOL_MODE ) {
                bdb->bi_dbenv_xflags |= DB_TXN_NOT_DURABLE;
 
+       /* slapindex algorithm tuning */
+       } else if ( strcasecmp( argv[0], "linearindex" ) == 0 ) {
+               bdb->bi_linear_index = 1;
+
        /* transaction checkpoint configuration */
        } else if ( strcasecmp( argv[0], "checkpoint" ) == 0 ) {
                if ( argc < 3 ) {

diff --git a/servers/slapd/back-bdb/tools.c b/servers/slapd/back-bdb/tools.c
index 6484e4db9e285d61ef2add639e2525312771b916..8898017156db93e9bbf0216b64b087be369fbaf3 100644 (file)
--- a/servers/slapd/back-bdb/tools.c
+++ b/servers/slapd/back-bdb/tools.c
@@ -19,6 +19,7 @@
 #include <stdio.h>
 #include <ac/string.h>
 
+#define AVL_INTERNAL
 #include "back-bdb.h"
 #include "external.h"
 
@@ -35,6 +36,8 @@ static dn_id hbuf[HOLE_SIZE], *holes = hbuf;
 static unsigned nhmax = HOLE_SIZE;
 static unsigned nholes;
 
+Avlnode *index_attrs, index_dummy;
+
 int bdb_tool_entry_open(
        BackendDB *be, int mode )
 {
@@ -79,6 +82,8 @@ int bdb_tool_entry_close(
        return 0;
 }
 
+static int bdb_reindex_cmp(const void *a, const void *b) { return 0; }
+
 ID bdb_tool_entry_next(
        BackendDB *be )
 {
@@ -90,6 +95,7 @@ ID bdb_tool_entry_next(
        assert( slapMode & SLAP_TOOL_MODE );
        assert( bdb != NULL );
        
+       /* Initialization */
        if (cursor == NULL) {
                rc = bdb->bi_id2entry->bdi_db->cursor(
                        bdb->bi_id2entry->bdi_db, NULL, &cursor,
@@ -102,7 +108,22 @@ ID bdb_tool_entry_next(
        rc = cursor->c_get( cursor, &key, &data, DB_NEXT );
 
        if( rc != 0 ) {
-               return NOID;
+               /* If we're doing linear indexing and there are more attrs to
+                * index, and we're at the end of the database, start over.
+                */
+               if ( bdb->bi_attrs == &index_dummy ) {
+                       if ( index_attrs && rc == DB_NOTFOUND ) {
+                               /* optional - do a checkpoint here? */
+                               index_dummy.avl_data = avl_delete(&index_attrs, NULL, bdb_reindex_cmp);
+                               rc = cursor->c_get( cursor, &key, &data, DB_FIRST );
+                       }
+                       if ( rc ) {
+                               bdb->bi_attrs = NULL;
+                               return NOID;
+                       }
+               } else {
+                       return NOID;
+               }
        }
 
        if( data.data == NULL ) {
@@ -354,7 +375,8 @@ ID bdb_tool_entry_put(
                goto done;
        }
 
-       rc = bdb_index_entry_add( &op, tid, e );
+       if ( !bdb->bi_linear_index )
+               rc = bdb_index_entry_add( &op, tid, e );
        if( rc != 0 ) {
                snprintf( text->bv_val, text->bv_len,
                                "index_entry_add failed: %s (%d)",
@@ -424,6 +446,20 @@ int bdb_tool_entry_reindex(
                (long) id, 0, 0 );
 #endif
 
+       /* No indexes configured, nothing to do. Could return an
+        * error here to shortcut things.
+        */
+       if (!bi->bi_attrs) {
+               return 0;
+       }
+
+       /* Get the first attribute to index */
+       if (bi->bi_linear_index && !index_attrs && bi->bi_attrs != &index_dummy) {
+               index_attrs = bi->bi_attrs;
+               bi->bi_attrs = &index_dummy;
+               index_dummy.avl_data = avl_delete(&index_attrs, NULL, bdb_reindex_cmp);
+       }
+
        e = bdb_tool_entry_get( be, id );
 
        if( e == NULL ) {
@@ -472,7 +508,7 @@ int bdb_tool_entry_reindex(
        op.o_tmpmemctx = NULL;
        op.o_tmpmfuncs = &ch_mfuncs;
 
-#ifndef BDB_HIER
+#if 0 /* ndef BDB_HIER */
        /* add dn2id indices */
        rc = bdb_dn2id_add( &op, tid, NULL, e );
        if( rc != 0 && rc != DB_KEYEXIST ) {