X-Git-Url: https://git.sur5r.net/?a=blobdiff_plain;f=servers%2Fslapd%2Fback-bdb%2Ftools.c;h=65d947b379e5c6d94a326fe16abd6cfe6c1baa0f;hb=87466fc7c89475e1f6882cca6b3754a0c0b6b55b;hp=714adf7a6e5e8426660b3a6a6a31a5fa084c0c87;hpb=d85cb6efbbf262b91a773294a8d80e12bcb9d48a;p=openldap diff --git a/servers/slapd/back-bdb/tools.c b/servers/slapd/back-bdb/tools.c index 714adf7a6e..65d947b379 100644 --- a/servers/slapd/back-bdb/tools.c +++ b/servers/slapd/back-bdb/tools.c @@ -2,7 +2,7 @@ /* $OpenLDAP$ */ /* This work is part of OpenLDAP Software . * - * Copyright 2000-2005 The OpenLDAP Foundation. + * Copyright 2000-2007 The OpenLDAP Foundation. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -18,6 +18,7 @@ #include #include +#include #define AVL_INTERNAL #include "back-bdb.h" @@ -25,6 +26,8 @@ static DBC *cursor = NULL; static DBT key, data; +static EntryHeader eh; +static int eoff; typedef struct dn_id { ID id; @@ -38,13 +41,12 @@ static unsigned nholes; static int index_nattrs; +#ifdef BDB_TOOL_IDL_CACHING #define bdb_tool_idl_cmp BDB_SYMBOL(tool_idl_cmp) #define bdb_tool_idl_flush_one BDB_SYMBOL(tool_idl_flush_one) #define bdb_tool_idl_flush BDB_SYMBOL(tool_idl_flush) static int bdb_tool_idl_flush( BackendDB *be ); -static int bdb_tool_ix_rec( int base ); -static void * bdb_tool_index_task( void *ctx, void *ptr ); #define IDBLOCK 1024 @@ -61,14 +63,22 @@ typedef struct bdb_tool_idl_cache { } bdb_tool_idl_cache; static bdb_tool_idl_cache_entry *bdb_tool_idl_free_list; +#endif /* BDB_TOOL_IDL_CACHING */ static ID bdb_tool_ix_id; static Operation *bdb_tool_ix_op; -static volatile int *bdb_tool_index_threads; +static int *bdb_tool_index_threads, bdb_tool_index_tcount; static void *bdb_tool_index_rec; static struct bdb_info *bdb_tool_info; static ldap_pvt_thread_mutex_t bdb_tool_index_mutex; -static ldap_pvt_thread_cond_t bdb_tool_index_cond; +static ldap_pvt_thread_cond_t bdb_tool_index_cond_main; +static ldap_pvt_thread_cond_t bdb_tool_index_cond_work; + +static ldap_pvt_thread_mutex_t bdb_tool_trickle_mutex; +static ldap_pvt_thread_cond_t bdb_tool_trickle_cond; + +static void * bdb_tool_index_task( void *ctx, void *ptr ); +static void * bdb_tool_trickle_task( void *ctx, void *ptr ); int bdb_tool_entry_open( BackendDB *be, int mode ) @@ -79,7 +89,7 @@ int bdb_tool_entry_open( DBTzero( &key ); DBTzero( &data ); key.flags = DB_DBT_REALLOC; - data.flags = DB_DBT_REALLOC; + data.flags = DB_DBT_USERMEM; if (cursor == NULL) { int rc = bdb->bi_id2entry->bdi_db->cursor( @@ -90,23 +100,30 @@ int bdb_tool_entry_open( } } - /* Set up for slapindex */ - if ( !(slapMode & SLAP_TOOL_READONLY )) { - int i; - if ( !bdb_tool_info && ( slapMode & SLAP_TOOL_QUICK )) { + /* Set up for threaded slapindex */ + if (( slapMode & (SLAP_TOOL_QUICK|SLAP_TOOL_READONLY)) == SLAP_TOOL_QUICK ) { + if ( !bdb_tool_info ) { + ldap_pvt_thread_mutex_init( &bdb_tool_trickle_mutex ); + ldap_pvt_thread_cond_init( &bdb_tool_trickle_cond ); + ldap_pvt_thread_pool_submit( &connection_pool, bdb_tool_trickle_task, bdb->bi_dbenv ); + ldap_pvt_thread_mutex_init( &bdb_tool_index_mutex ); - ldap_pvt_thread_cond_init( &bdb_tool_index_cond ); - bdb_tool_index_threads = ch_malloc( slap_tool_thread_max * sizeof( int )); - bdb_tool_index_rec = ch_malloc( bdb->bi_nattrs * sizeof( IndexRec )); - for (i=1; ibi_nattrs ) { + int i; + bdb_tool_index_threads = ch_malloc( slap_tool_thread_max * sizeof( int )); + bdb_tool_index_rec = ch_malloc( bdb->bi_nattrs * sizeof( IndexRec )); + bdb_tool_index_tcount = slap_tool_thread_max - 1; + for (i=1; ibe_private; - if ( bdb_tool_info ) { slapd_shutdown = 1; + ldap_pvt_thread_mutex_lock( &bdb_tool_trickle_mutex ); + ldap_pvt_thread_cond_signal( &bdb_tool_trickle_cond ); + ldap_pvt_thread_mutex_unlock( &bdb_tool_trickle_mutex ); ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex ); - ldap_pvt_thread_cond_broadcast( &bdb_tool_index_cond ); + bdb_tool_index_tcount = slap_tool_thread_max - 1; + ldap_pvt_thread_cond_broadcast( &bdb_tool_index_cond_work ); ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex ); } @@ -128,9 +147,9 @@ int bdb_tool_entry_close( ch_free( key.data ); key.data = NULL; } - if( data.data ) { - ch_free( data.data ); - data.data = NULL; + if( eh.bv.bv_val ) { + ch_free( eh.bv.bv_val ); + eh.bv.bv_val = NULL; } if( cursor ) { @@ -138,7 +157,9 @@ int bdb_tool_entry_close( cursor = NULL; } +#ifdef BDB_TOOL_IDL_CACHING bdb_tool_idl_flush( be ); +#endif if( nholes ) { unsigned i; @@ -159,14 +180,19 @@ ID bdb_tool_entry_next( int rc; ID id; struct bdb_info *bdb = (struct bdb_info *) be->be_private; + char buf[16], *dptr; assert( be != NULL ); assert( slapMode & SLAP_TOOL_MODE ); assert( bdb != NULL ); + /* Get the header */ + data.ulen = data.dlen = sizeof( buf ); + data.data = buf; + data.flags |= DB_DBT_PARTIAL; rc = cursor->c_get( cursor, &key, &data, DB_NEXT ); - if( rc != 0 ) { + if( rc ) { /* If we're doing linear indexing and there are more attrs to * index, and we're at the end of the database, start over. */ @@ -184,7 +210,13 @@ ID bdb_tool_entry_next( } } - if( data.data == NULL ) { + dptr = eh.bv.bv_val; + eh.bv.bv_val = buf; + eh.bv.bv_len = data.size; + rc = entry_header( &eh ); + eoff = eh.data - eh.bv.bv_val; + eh.bv.bv_val = dptr; + if( rc ) { return NOID; } @@ -244,46 +276,62 @@ Entry* bdb_tool_entry_get( BackendDB *be, ID id ) { int rc; Entry *e = NULL; - struct berval bv; assert( be != NULL ); assert( slapMode & SLAP_TOOL_MODE ); - assert( data.data != NULL ); - DBT2bv( &data, &bv ); + /* Get the size */ + data.flags ^= DB_DBT_PARTIAL; + data.ulen = 0; + rc = cursor->c_get( cursor, &key, &data, DB_CURRENT ); + if ( rc != DB_BUFFER_SMALL ) goto leave; + + /* Allocate a block and retrieve the data */ + eh.bv.bv_len = eh.nvals * sizeof( struct berval ) + data.size; + eh.bv.bv_val = ch_realloc( eh.bv.bv_val, eh.bv.bv_len ); + eh.data = eh.bv.bv_val + eh.nvals * sizeof( struct berval ); + data.data = eh.data; + data.ulen = data.size; + + /* Skip past already parsed nattr/nvals */ + eh.data += eoff; + + rc = cursor->c_get( cursor, &key, &data, DB_CURRENT ); + if ( rc ) goto leave; #ifdef SLAP_ZONE_ALLOC /* FIXME: will add ctx later */ - rc = entry_decode( &bv, &e, NULL ); + rc = entry_decode( &eh, &e, NULL ); #else - rc = entry_decode( &bv, &e ); + rc = entry_decode( &eh, &e ); #endif if( rc == LDAP_SUCCESS ) { e->e_id = id; - } #ifdef BDB_HIER - if ( slapMode & SLAP_TOOL_READONLY ) { - EntryInfo *ei = NULL; - Operation op = {0}; - Opheader ohdr = {0}; - - op.o_hdr = &ohdr; - op.o_bd = be; - op.o_tmpmemctx = NULL; - op.o_tmpmfuncs = &ch_mfuncs; - - rc = bdb_cache_find_parent( &op, NULL, cursor->locker, id, &ei ); - if ( rc == LDAP_SUCCESS ) { - bdb_cache_entryinfo_unlock( ei ); - e->e_private = ei; - ei->bei_e = e; - bdb_fix_dn( e, 0 ); - ei->bei_e = NULL; - e->e_private = NULL; + if ( slapMode & SLAP_TOOL_READONLY ) { + EntryInfo *ei = NULL; + Operation op = {0}; + Opheader ohdr = {0}; + + op.o_hdr = &ohdr; + op.o_bd = be; + op.o_tmpmemctx = NULL; + op.o_tmpmfuncs = &ch_mfuncs; + + rc = bdb_cache_find_parent( &op, NULL, cursor->locker, id, &ei ); + if ( rc == LDAP_SUCCESS ) { + bdb_cache_entryinfo_unlock( ei ); + e->e_private = ei; + ei->bei_e = e; + bdb_fix_dn( e, 0 ); + ei->bei_e = NULL; + e->e_private = NULL; + } } - } #endif + } +leave: return e; } @@ -387,6 +435,9 @@ bdb_tool_index_add( { struct bdb_info *bdb = (struct bdb_info *) op->o_bd->be_private; + if ( !bdb->bi_nattrs ) + return 0; + if ( slapMode & SLAP_TOOL_QUICK ) { IndexRec *ir; int i, rc; @@ -404,23 +455,34 @@ bdb_tool_index_add( bdb_tool_ix_id = e->e_id; bdb_tool_ix_op = op; ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex ); + /* Wait for all threads to be ready */ + while ( bdb_tool_index_tcount ) { + ldap_pvt_thread_cond_wait( &bdb_tool_index_cond_main, + &bdb_tool_index_mutex ); + } for ( i=1; ie_id, 0 ); if ( rc ) return rc; + ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex ); for ( i=1; ie_id & 0xfff ) == 0xfff )) { + ldap_pvt_thread_mutex_lock( &bdb_tool_trickle_mutex ); + ldap_pvt_thread_cond_signal( &bdb_tool_trickle_cond ); + ldap_pvt_thread_mutex_unlock( &bdb_tool_trickle_mutex ); + } + + if ( !bdb->bi_linear_index ) + rc = bdb_tool_index_add( &op, tid, e ); if( rc != 0 ) { snprintf( text->bv_val, text->bv_len, - "id2entry_add failed: %s (%d)", + "index_entry_add failed: %s (%d)", db_strerror(rc), rc ); Debug( LDAP_DEBUG_ANY, "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n", @@ -484,11 +552,11 @@ ID bdb_tool_entry_put( goto done; } - if ( !bdb->bi_linear_index ) - rc = bdb_tool_index_add( &op, tid, e ); + /* id2entry index */ + rc = bdb_id2entry_add( be, tid, e ); if( rc != 0 ) { snprintf( text->bv_val, text->bv_len, - "index_entry_add failed: %s (%d)", + "id2entry_add failed: %s (%d)", db_strerror(rc), rc ); Debug( LDAP_DEBUG_ANY, "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n", @@ -529,7 +597,8 @@ done: int bdb_tool_entry_reindex( BackendDB *be, - ID id ) + ID id, + AttributeDescription **adv ) { struct bdb_info *bi = (struct bdb_info *) be->be_private; int rc; @@ -549,6 +618,47 @@ int bdb_tool_entry_reindex( return 0; } + /* Check for explicit list of attrs to index */ + if ( adv ) { + int i, j, n; + + if ( bi->bi_attrs[0]->ai_desc != adv[0] ) { + /* count */ + for ( n = 0; adv[n]; n++ ) ; + + /* insertion sort */ + for ( i = 0; i < n; i++ ) { + AttributeDescription *ad = adv[i]; + for ( j = i-1; j>=0; j--) { + if ( SLAP_PTRCMP( adv[j], ad ) <= 0 ) break; + adv[j+1] = adv[j]; + } + adv[j+1] = ad; + } + } + + for ( i = 0; adv[i]; i++ ) { + if ( bi->bi_attrs[i]->ai_desc != adv[i] ) { + for ( j = i+1; j < bi->bi_nattrs; j++ ) { + if ( bi->bi_attrs[j]->ai_desc == adv[i] ) { + AttrInfo *ai = bi->bi_attrs[i]; + bi->bi_attrs[i] = bi->bi_attrs[j]; + bi->bi_attrs[j] = ai; + break; + } + } + if ( j == bi->bi_nattrs ) { + Debug( LDAP_DEBUG_ANY, + LDAP_XSTRING(bdb_tool_entry_reindex) + ": no index configured for %s\n", + adv[i]->ad_cname.bv_val, 0, 0 ); + return -1; + } + } + } + bi->bi_nattrs = i; + } + /* Get the first attribute to index */ if (bi->bi_linear_index && !index_nattrs) { index_nattrs = bi->bi_nattrs - 1; @@ -677,33 +787,6 @@ ID bdb_tool_entry_modify( goto done; } -#if 0 - /* FIXME: this is bogus, we don't have the old values to delete - * from the index because the given entry has already been modified. - */ - rc = bdb_index_entry_del( &op, tid, e ); - if( rc != 0 ) { - snprintf( text->bv_val, text->bv_len, - "index_entry_del failed: %s (%d)", - db_strerror(rc), rc ); - Debug( LDAP_DEBUG_ANY, - "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n", - text->bv_val, 0, 0 ); - goto done; - } -#endif - - rc = bdb_index_entry_add( &op, tid, e ); - if( rc != 0 ) { - snprintf( text->bv_val, text->bv_len, - "index_entry_add failed: %s (%d)", - db_strerror(rc), rc ); - Debug( LDAP_DEBUG_ANY, - "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n", - text->bv_val, 0, 0 ); - goto done; - } - done: if( rc == 0 ) { if (! (slapMode & SLAP_TOOL_QUICK)) { @@ -735,7 +818,7 @@ done: return e->e_id; } - +#ifdef BDB_TOOL_IDL_CACHING static int bdb_tool_idl_cmp( const void *v1, const void *v2 ) { @@ -758,8 +841,8 @@ bdb_tool_idl_flush_one( void *v1, void *arg ) int i, rc; ID id, nid; - if ( !ic->count ) { - ch_free( ic ); + /* Freshly allocated, ignore it */ + if ( !ic->head && ic->count <= BDB_IDL_DB_SIZE ) { return 0; } @@ -852,22 +935,31 @@ bdb_tool_idl_flush_one( void *v1, void *arg ) } } if ( ic->head ) { + ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock ); ic->tail->next = bdb_tool_idl_free_list; bdb_tool_idl_free_list = ic->head; bdb->bi_idl_cache_size -= n; + ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock ); } } - ch_free( ic ); + if ( ic != db->app_private ) { + ch_free( ic ); + } else { + ic->head = ic->tail = NULL; + } curs->c_close( curs ); return rc; } static int -bdb_tool_idl_flush_db( DB *db ) +bdb_tool_idl_flush_db( DB *db, bdb_tool_idl_cache *ic ) { - int rc = avl_apply( db->app_private, bdb_tool_idl_flush_one, db, -1, - AVL_INORDER ); - avl_free( db->app_private, NULL ); + Avlnode *root = db->app_private; + int rc; + + db->app_private = ic; + rc = avl_apply( root, bdb_tool_idl_flush_one, db, -1, AVL_INORDER ); + avl_free( root, NULL ); db->app_private = NULL; if ( rc != -1 ) rc = 0; @@ -885,7 +977,7 @@ bdb_tool_idl_flush( BackendDB *be ) for ( i=BDB_NDB; i < bdb->bi_ndatabases; i++ ) { db = bdb->bi_databases[i]->bdi_db; if ( !db->app_private ) continue; - rc = bdb_tool_idl_flush_db( db ); + rc = bdb_tool_idl_flush_db( db, NULL ); if ( rc ) break; } @@ -905,18 +997,14 @@ int bdb_tool_idl_add( struct bdb_info *bdb = (struct bdb_info *) be->be_private; bdb_tool_idl_cache *ic, itmp; bdb_tool_idl_cache_entry *ice; - Avlnode *root; - int rc, do_retry = 1; + int rc; if ( !bdb->bi_idl_cache_max_size ) return bdb_idl_insert_key( be, db, txn, key, id ); DBT2bv( key, &itmp.kstr ); -retry: - root = db->app_private; - - ic = avl_find( root, &itmp, bdb_tool_idl_cmp ); + ic = avl_find( (Avlnode *)db->app_private, &itmp, bdb_tool_idl_cmp ); /* No entry yet, create one */ if ( !ic ) { @@ -932,8 +1020,8 @@ retry: ic->head = ic->tail = NULL; ic->last = 0; ic->count = 0; - avl_insert( &root, ic, bdb_tool_idl_cmp, avl_dup_error ); - db->app_private = root; + avl_insert( (Avlnode **)&db->app_private, ic, bdb_tool_idl_cmp, + avl_dup_error ); /* load existing key count here */ rc = db->cursor( db, NULL, &curs, 0 ); @@ -979,21 +1067,24 @@ retry: } /* No free block, create that too */ if ( !ic->tail || ( ic->count & (IDBLOCK-1)) == 0) { + ice = NULL; ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock ); - if ( do_retry && - bdb->bi_idl_cache_size >= bdb->bi_idl_cache_max_size ) { - do_retry = 0; - rc = bdb_tool_idl_flush_db( db ); + if ( bdb->bi_idl_cache_size >= bdb->bi_idl_cache_max_size ) { ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock ); + rc = bdb_tool_idl_flush_db( db, ic ); if ( rc ) return rc; - goto retry; + avl_insert( (Avlnode **)&db->app_private, ic, bdb_tool_idl_cmp, + avl_dup_error ); + ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock ); } bdb->bi_idl_cache_size++; if ( bdb_tool_idl_free_list ) { ice = bdb_tool_idl_free_list; bdb_tool_idl_free_list = ice->next; - } else { + } + ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock ); + if ( !ice ) { ice = ch_malloc( sizeof( bdb_tool_idl_cache_entry )); } memset( ice, 0, sizeof( *ice )); @@ -1003,7 +1094,6 @@ retry: ic->tail->next = ice; } ic->tail = ice; - ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock ); if ( !ic->count ) ic->first = id; } @@ -1013,6 +1103,24 @@ retry: return 0; } +#endif + +static void * +bdb_tool_trickle_task( void *ctx, void *ptr ) +{ + DB_ENV *env = ptr; + int wrote; + + ldap_pvt_thread_mutex_lock( &bdb_tool_trickle_mutex ); + while ( 1 ) { + ldap_pvt_thread_cond_wait( &bdb_tool_trickle_cond, + &bdb_tool_trickle_mutex ); + if ( slapd_shutdown ) + break; + env->memp_trickle( env, 30, &wrote ); + } + ldap_pvt_thread_mutex_unlock( &bdb_tool_trickle_mutex ); +} static void * bdb_tool_index_task( void *ctx, void *ptr ) @@ -1022,7 +1130,10 @@ bdb_tool_index_task( void *ctx, void *ptr ) free( ptr ); while ( 1 ) { ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex ); - ldap_pvt_thread_cond_wait( &bdb_tool_index_cond, + bdb_tool_index_tcount--; + if ( !bdb_tool_index_tcount ) + ldap_pvt_thread_cond_signal( &bdb_tool_index_cond_main ); + ldap_pvt_thread_cond_wait( &bdb_tool_index_cond_work, &bdb_tool_index_mutex ); ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex ); if ( slapd_shutdown )