1 /* tools.c - tools for slap tools */
3 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
5 * Copyright 2000-2007 The OpenLDAP Foundation.
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted only as authorized by the OpenLDAP
12 * A copy of this license is available in the file LICENSE in the
13 * top-level directory of the distribution or, alternatively, at
14 * <http://www.OpenLDAP.org/license.html>.
20 #include <ac/string.h>
27 static DBC *cursor = NULL;
29 static EntryHeader eh;
32 typedef struct dn_id {
37 #define HOLE_SIZE 4096
38 static dn_id hbuf[HOLE_SIZE], *holes = hbuf;
39 static unsigned nhmax = HOLE_SIZE;
40 static unsigned nholes;
42 static int index_nattrs;
44 #ifdef BDB_TOOL_IDL_CACHING
45 #define bdb_tool_idl_cmp BDB_SYMBOL(tool_idl_cmp)
46 #define bdb_tool_idl_flush_one BDB_SYMBOL(tool_idl_flush_one)
47 #define bdb_tool_idl_flush BDB_SYMBOL(tool_idl_flush)
49 static int bdb_tool_idl_flush( BackendDB *be );
53 typedef struct bdb_tool_idl_cache_entry {
54 struct bdb_tool_idl_cache_entry *next;
56 } bdb_tool_idl_cache_entry;
58 typedef struct bdb_tool_idl_cache {
60 bdb_tool_idl_cache_entry *head, *tail;
65 static bdb_tool_idl_cache_entry *bdb_tool_idl_free_list;
66 #endif /* BDB_TOOL_IDL_CACHING */
68 static ID bdb_tool_ix_id;
69 static Operation *bdb_tool_ix_op;
70 static int *bdb_tool_index_threads, bdb_tool_index_tcount;
71 static void *bdb_tool_index_rec;
72 static struct bdb_info *bdb_tool_info;
73 static ldap_pvt_thread_mutex_t bdb_tool_index_mutex;
74 static ldap_pvt_thread_cond_t bdb_tool_index_cond_main;
75 static ldap_pvt_thread_cond_t bdb_tool_index_cond_work;
77 static ldap_pvt_thread_mutex_t bdb_tool_trickle_mutex;
78 static ldap_pvt_thread_cond_t bdb_tool_trickle_cond;
80 static void * bdb_tool_index_task( void *ctx, void *ptr );
81 static void * bdb_tool_trickle_task( void *ctx, void *ptr );
83 int bdb_tool_entry_open(
84 BackendDB *be, int mode )
86 struct bdb_info *bdb = (struct bdb_info *) be->be_private;
88 /* initialize key and data thangs */
91 key.flags = DB_DBT_REALLOC;
92 data.flags = DB_DBT_USERMEM;
95 int rc = bdb->bi_id2entry->bdi_db->cursor(
96 bdb->bi_id2entry->bdi_db, NULL, &cursor,
103 /* Set up for threaded slapindex */
104 if (( slapMode & (SLAP_TOOL_QUICK|SLAP_TOOL_READONLY)) == SLAP_TOOL_QUICK ) {
105 if ( !bdb_tool_info ) {
106 ldap_pvt_thread_mutex_init( &bdb_tool_trickle_mutex );
107 ldap_pvt_thread_cond_init( &bdb_tool_trickle_cond );
108 ldap_pvt_thread_pool_submit( &connection_pool, bdb_tool_trickle_task, bdb->bi_dbenv );
110 ldap_pvt_thread_mutex_init( &bdb_tool_index_mutex );
111 ldap_pvt_thread_cond_init( &bdb_tool_index_cond_main );
112 ldap_pvt_thread_cond_init( &bdb_tool_index_cond_work );
113 if ( bdb->bi_nattrs ) {
115 bdb_tool_index_threads = ch_malloc( slap_tool_thread_max * sizeof( int ));
116 bdb_tool_index_rec = ch_malloc( bdb->bi_nattrs * sizeof( IndexRec ));
117 bdb_tool_index_tcount = slap_tool_thread_max - 1;
118 for (i=1; i<slap_tool_thread_max; i++) {
119 int *ptr = ch_malloc( sizeof( int ));
121 ldap_pvt_thread_pool_submit( &connection_pool,
122 bdb_tool_index_task, ptr );
132 int bdb_tool_entry_close(
135 if ( bdb_tool_info ) {
137 ldap_pvt_thread_mutex_lock( &bdb_tool_trickle_mutex );
138 ldap_pvt_thread_cond_signal( &bdb_tool_trickle_cond );
139 ldap_pvt_thread_mutex_unlock( &bdb_tool_trickle_mutex );
140 ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex );
141 bdb_tool_index_tcount = slap_tool_thread_max - 1;
142 ldap_pvt_thread_cond_broadcast( &bdb_tool_index_cond_work );
143 ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex );
151 ch_free( eh.bv.bv_val );
156 cursor->c_close( cursor );
160 #ifdef BDB_TOOL_IDL_CACHING
161 bdb_tool_idl_flush( be );
166 fprintf( stderr, "Error, entries missing!\n");
167 for (i=0; i<nholes; i++) {
168 fprintf(stderr, " entry %ld: %s\n",
169 holes[i].id, holes[i].dn.bv_val);
177 ID bdb_tool_entry_next(
182 struct bdb_info *bdb = (struct bdb_info *) be->be_private;
185 assert( be != NULL );
186 assert( slapMode & SLAP_TOOL_MODE );
187 assert( bdb != NULL );
190 data.ulen = data.dlen = sizeof( buf );
192 data.flags |= DB_DBT_PARTIAL;
193 rc = cursor->c_get( cursor, &key, &data, DB_NEXT );
196 /* If we're doing linear indexing and there are more attrs to
197 * index, and we're at the end of the database, start over.
199 if ( index_nattrs && rc == DB_NOTFOUND ) {
200 /* optional - do a checkpoint here? */
201 bdb_attr_info_free( bdb->bi_attrs[0] );
202 bdb->bi_attrs[0] = bdb->bi_attrs[index_nattrs];
204 rc = cursor->c_get( cursor, &key, &data, DB_FIRST );
215 eh.bv.bv_len = data.size;
216 rc = entry_header( &eh );
217 eoff = eh.data - eh.bv.bv_val;
223 BDB_DISK2ID( key.data, &id );
227 ID bdb_tool_dn2id_get(
234 EntryInfo *ei = NULL;
237 if ( BER_BVISEMPTY(dn) )
242 op.o_tmpmemctx = NULL;
243 op.o_tmpmfuncs = &ch_mfuncs;
245 rc = bdb_cache_find_ndn( &op, NULL, dn, &ei );
246 if ( ei ) bdb_cache_entryinfo_unlock( ei );
247 if ( rc == DB_NOTFOUND )
253 int bdb_tool_id2entry_get(
259 int rc = bdb_id2entry( be, NULL, 0, id, e );
261 if ( rc == DB_NOTFOUND && id == 0 ) {
262 Entry *dummy = ch_calloc( 1, sizeof(Entry) );
263 struct berval gluebv = BER_BVC("glue");
264 dummy->e_name.bv_val = ch_strdup( "" );
265 dummy->e_nname.bv_val = ch_strdup( "" );
266 attr_merge_one( dummy, slap_schema.si_ad_objectClass, &gluebv, NULL );
267 attr_merge_one( dummy, slap_schema.si_ad_structuralObjectClass,
275 Entry* bdb_tool_entry_get( BackendDB *be, ID id )
280 assert( be != NULL );
281 assert( slapMode & SLAP_TOOL_MODE );
284 data.flags ^= DB_DBT_PARTIAL;
286 rc = cursor->c_get( cursor, &key, &data, DB_CURRENT );
287 if ( rc != DB_BUFFER_SMALL ) goto done;
289 /* Allocate a block and retrieve the data */
290 eh.bv.bv_len = eh.nvals * sizeof( struct berval ) + data.size;
291 eh.bv.bv_val = ch_realloc( eh.bv.bv_val, eh.bv.bv_len );
292 eh.data = eh.bv.bv_val + eh.nvals * sizeof( struct berval );
294 data.ulen = data.size;
296 /* Skip past already parsed nattr/nvals */
299 rc = cursor->c_get( cursor, &key, &data, DB_CURRENT );
302 #ifdef SLAP_ZONE_ALLOC
303 /* FIXME: will add ctx later */
304 rc = entry_decode( &eh, &e, NULL );
306 rc = entry_decode( &eh, &e );
309 if( rc == LDAP_SUCCESS ) {
312 if ( slapMode & SLAP_TOOL_READONLY ) {
313 EntryInfo *ei = NULL;
319 op.o_tmpmemctx = NULL;
320 op.o_tmpmfuncs = &ch_mfuncs;
322 rc = bdb_cache_find_parent( &op, NULL, cursor->locker, id, &ei );
323 if ( rc == LDAP_SUCCESS ) {
324 bdb_cache_entryinfo_unlock( ei );
338 static int bdb_tool_next_id(
345 struct berval dn = e->e_name;
346 struct berval ndn = e->e_nname;
347 struct berval pdn, npdn;
348 EntryInfo *ei = NULL, eidummy;
351 if (ndn.bv_len == 0) {
356 rc = bdb_cache_find_ndn( op, tid, &ndn, &ei );
357 if ( ei ) bdb_cache_entryinfo_unlock( ei );
358 if ( rc == DB_NOTFOUND ) {
359 if ( !be_issuffix( op->o_bd, &ndn ) ) {
361 dnParent( &dn, &pdn );
362 dnParent( &ndn, &npdn );
365 rc = bdb_tool_next_id( op, tid, e, text, 1 );
371 /* If parent didn't exist, it was created just now
372 * and its ID is now in e->e_id. Make sure the current
373 * entry gets added under the new parent ID.
375 if ( eid != e->e_id ) {
376 eidummy.bei_id = e->e_id;
380 rc = bdb_next_id( op->o_bd, tid, &e->e_id );
382 snprintf( text->bv_val, text->bv_len,
383 "next_id failed: %s (%d)",
384 db_strerror(rc), rc );
385 Debug( LDAP_DEBUG_ANY,
386 "=> bdb_tool_next_id: %s\n", text->bv_val, 0, 0 );
389 rc = bdb_dn2id_add( op, tid, ei, e );
391 snprintf( text->bv_val, text->bv_len,
392 "dn2id_add failed: %s (%d)",
393 db_strerror(rc), rc );
394 Debug( LDAP_DEBUG_ANY,
395 "=> bdb_tool_next_id: %s\n", text->bv_val, 0, 0 );
397 if ( nholes == nhmax - 1 ) {
398 if ( holes == hbuf ) {
399 holes = ch_malloc( nhmax * sizeof(dn_id) * 2 );
400 AC_MEMCPY( holes, hbuf, sizeof(hbuf) );
402 holes = ch_realloc( holes, nhmax * sizeof(dn_id) * 2 );
406 ber_dupbv( &holes[nholes].dn, &ndn );
407 holes[nholes++].id = e->e_id;
409 } else if ( !hole ) {
412 e->e_id = ei->bei_id;
414 for ( i=0; i<nholes; i++) {
415 if ( holes[i].id == e->e_id ) {
417 free(holes[i].dn.bv_val);
418 for (j=i;j<nholes;j++) holes[j] = holes[j+1];
422 } else if ( holes[i].id > e->e_id ) {
436 struct bdb_info *bdb = (struct bdb_info *) op->o_bd->be_private;
438 if ( !bdb->bi_nattrs )
441 if ( slapMode & SLAP_TOOL_QUICK ) {
446 ir = bdb_tool_index_rec;
447 memset(ir, 0, bdb->bi_nattrs * sizeof( IndexRec ));
449 for ( a = e->e_attrs; a != NULL; a = a->a_next ) {
450 rc = bdb_index_recset( bdb, a, a->a_desc->ad_type,
451 &a->a_desc->ad_tags, ir );
455 bdb_tool_ix_id = e->e_id;
457 ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex );
458 /* Wait for all threads to be ready */
459 while ( bdb_tool_index_tcount ) {
460 ldap_pvt_thread_cond_wait( &bdb_tool_index_cond_main,
461 &bdb_tool_index_mutex );
463 for ( i=1; i<slap_tool_thread_max; i++ )
464 bdb_tool_index_threads[i] = LDAP_BUSY;
465 bdb_tool_index_tcount = slap_tool_thread_max - 1;
466 ldap_pvt_thread_cond_broadcast( &bdb_tool_index_cond_work );
467 ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex );
468 rc = bdb_index_recrun( op, bdb, ir, e->e_id, 0 );
471 ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex );
472 for ( i=1; i<slap_tool_thread_max; i++ ) {
473 if ( bdb_tool_index_threads[i] == LDAP_BUSY ) {
474 ldap_pvt_thread_cond_wait( &bdb_tool_index_cond_main,
475 &bdb_tool_index_mutex );
479 if ( bdb_tool_index_threads[i] ) {
480 rc = bdb_tool_index_threads[i];
484 ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex );
487 return bdb_index_entry_add( op, txn, e );
491 ID bdb_tool_entry_put(
494 struct berval *text )
497 struct bdb_info *bdb = (struct bdb_info *) be->be_private;
502 assert( be != NULL );
503 assert( slapMode & SLAP_TOOL_MODE );
505 assert( text != NULL );
506 assert( text->bv_val != NULL );
507 assert( text->bv_val[0] == '\0' ); /* overconservative? */
509 Debug( LDAP_DEBUG_TRACE, "=> " LDAP_XSTRING(bdb_tool_entry_put)
510 "( %ld, \"%s\" )\n", (long) e->e_id, e->e_dn, 0 );
512 if (! (slapMode & SLAP_TOOL_QUICK)) {
513 rc = TXN_BEGIN( bdb->bi_dbenv, NULL, &tid,
514 bdb->bi_db_opflags );
516 snprintf( text->bv_val, text->bv_len,
517 "txn_begin failed: %s (%d)",
518 db_strerror(rc), rc );
519 Debug( LDAP_DEBUG_ANY,
520 "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
521 text->bv_val, 0, 0 );
528 op.o_tmpmemctx = NULL;
529 op.o_tmpmfuncs = &ch_mfuncs;
531 /* add dn2id indices */
532 rc = bdb_tool_next_id( &op, tid, e, text, 0 );
537 if (( slapMode & SLAP_TOOL_QUICK ) && (( e->e_id & 0xfff ) == 0xfff )) {
538 ldap_pvt_thread_mutex_lock( &bdb_tool_trickle_mutex );
539 ldap_pvt_thread_cond_signal( &bdb_tool_trickle_cond );
540 ldap_pvt_thread_mutex_unlock( &bdb_tool_trickle_mutex );
543 if ( !bdb->bi_linear_index )
544 rc = bdb_tool_index_add( &op, tid, e );
546 snprintf( text->bv_val, text->bv_len,
547 "index_entry_add failed: %s (%d)",
548 db_strerror(rc), rc );
549 Debug( LDAP_DEBUG_ANY,
550 "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
551 text->bv_val, 0, 0 );
556 rc = bdb_id2entry_add( be, tid, e );
558 snprintf( text->bv_val, text->bv_len,
559 "id2entry_add failed: %s (%d)",
560 db_strerror(rc), rc );
561 Debug( LDAP_DEBUG_ANY,
562 "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
563 text->bv_val, 0, 0 );
569 if ( !( slapMode & SLAP_TOOL_QUICK )) {
570 rc = TXN_COMMIT( tid, 0 );
572 snprintf( text->bv_val, text->bv_len,
573 "txn_commit failed: %s (%d)",
574 db_strerror(rc), rc );
575 Debug( LDAP_DEBUG_ANY,
576 "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
577 text->bv_val, 0, 0 );
583 if ( !( slapMode & SLAP_TOOL_QUICK )) {
585 snprintf( text->bv_val, text->bv_len,
586 "txn_aborted! %s (%d)",
587 db_strerror(rc), rc );
588 Debug( LDAP_DEBUG_ANY,
589 "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
590 text->bv_val, 0, 0 );
598 int bdb_tool_entry_reindex(
601 AttributeDescription **adv )
603 struct bdb_info *bi = (struct bdb_info *) be->be_private;
610 Debug( LDAP_DEBUG_ARGS,
611 "=> " LDAP_XSTRING(bdb_tool_entry_reindex) "( %ld )\n",
614 /* No indexes configured, nothing to do. Could return an
615 * error here to shortcut things.
621 /* Check for explicit list of attrs to index */
625 if ( bi->bi_attrs[0]->ai_desc != adv[0] ) {
627 for ( n = 0; adv[n]; n++ ) ;
630 for ( i = 0; i < n; i++ ) {
631 AttributeDescription *ad = adv[i];
632 for ( j = i-1; j>=0; j--) {
633 if ( SLAP_PTRCMP( adv[j], ad ) <= 0 ) break;
640 for ( i = 0; adv[i]; i++ ) {
641 if ( bi->bi_attrs[i]->ai_desc != adv[i] ) {
642 for ( j = i+1; j < bi->bi_nattrs; j++ ) {
643 if ( bi->bi_attrs[j]->ai_desc == adv[i] ) {
644 AttrInfo *ai = bi->bi_attrs[i];
645 bi->bi_attrs[i] = bi->bi_attrs[j];
646 bi->bi_attrs[j] = ai;
650 if ( j == bi->bi_nattrs ) {
651 Debug( LDAP_DEBUG_ANY,
652 LDAP_XSTRING(bdb_tool_entry_reindex)
653 ": no index configured for %s\n",
654 adv[i]->ad_cname.bv_val, 0, 0 );
662 /* Get the first attribute to index */
663 if (bi->bi_linear_index && !index_nattrs) {
664 index_nattrs = bi->bi_nattrs - 1;
668 e = bdb_tool_entry_get( be, id );
671 Debug( LDAP_DEBUG_ANY,
672 LDAP_XSTRING(bdb_tool_entry_reindex)
673 ": could not locate id=%ld\n",
678 if (! (slapMode & SLAP_TOOL_QUICK)) {
679 rc = TXN_BEGIN( bi->bi_dbenv, NULL, &tid, bi->bi_db_opflags );
681 Debug( LDAP_DEBUG_ANY,
682 "=> " LDAP_XSTRING(bdb_tool_entry_reindex) ": "
683 "txn_begin failed: %s (%d)\n",
684 db_strerror(rc), rc, 0 );
690 * just (re)add them for now
691 * assume that some other routine (not yet implemented)
692 * will zap index databases
696 Debug( LDAP_DEBUG_TRACE,
697 "=> " LDAP_XSTRING(bdb_tool_entry_reindex) "( %ld, \"%s\" )\n",
698 (long) id, e->e_dn, 0 );
702 op.o_tmpmemctx = NULL;
703 op.o_tmpmfuncs = &ch_mfuncs;
705 rc = bdb_tool_index_add( &op, tid, e );
709 if (! (slapMode & SLAP_TOOL_QUICK)) {
710 rc = TXN_COMMIT( tid, 0 );
712 Debug( LDAP_DEBUG_ANY,
713 "=> " LDAP_XSTRING(bdb_tool_entry_reindex)
714 ": txn_commit failed: %s (%d)\n",
715 db_strerror(rc), rc, 0 );
721 if (! (slapMode & SLAP_TOOL_QUICK)) {
723 Debug( LDAP_DEBUG_ANY,
724 "=> " LDAP_XSTRING(bdb_tool_entry_reindex)
725 ": txn_aborted! %s (%d)\n",
726 db_strerror(rc), rc, 0 );
730 bdb_entry_release( &op, e, 0 );
735 ID bdb_tool_entry_modify(
738 struct berval *text )
741 struct bdb_info *bdb = (struct bdb_info *) be->be_private;
746 assert( be != NULL );
747 assert( slapMode & SLAP_TOOL_MODE );
749 assert( text != NULL );
750 assert( text->bv_val != NULL );
751 assert( text->bv_val[0] == '\0' ); /* overconservative? */
753 assert ( e->e_id != NOID );
755 Debug( LDAP_DEBUG_TRACE,
756 "=> " LDAP_XSTRING(bdb_tool_entry_modify) "( %ld, \"%s\" )\n",
757 (long) e->e_id, e->e_dn, 0 );
759 if (! (slapMode & SLAP_TOOL_QUICK)) {
760 rc = TXN_BEGIN( bdb->bi_dbenv, NULL, &tid,
761 bdb->bi_db_opflags );
763 snprintf( text->bv_val, text->bv_len,
764 "txn_begin failed: %s (%d)",
765 db_strerror(rc), rc );
766 Debug( LDAP_DEBUG_ANY,
767 "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n",
768 text->bv_val, 0, 0 );
775 op.o_tmpmemctx = NULL;
776 op.o_tmpmfuncs = &ch_mfuncs;
779 rc = bdb_id2entry_update( be, tid, e );
781 snprintf( text->bv_val, text->bv_len,
782 "id2entry_add failed: %s (%d)",
783 db_strerror(rc), rc );
784 Debug( LDAP_DEBUG_ANY,
785 "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n",
786 text->bv_val, 0, 0 );
792 if (! (slapMode & SLAP_TOOL_QUICK)) {
793 rc = TXN_COMMIT( tid, 0 );
795 snprintf( text->bv_val, text->bv_len,
796 "txn_commit failed: %s (%d)",
797 db_strerror(rc), rc );
798 Debug( LDAP_DEBUG_ANY,
799 "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": "
800 "%s\n", text->bv_val, 0, 0 );
806 if (! (slapMode & SLAP_TOOL_QUICK)) {
808 snprintf( text->bv_val, text->bv_len,
809 "txn_aborted! %s (%d)",
810 db_strerror(rc), rc );
811 Debug( LDAP_DEBUG_ANY,
812 "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n",
813 text->bv_val, 0, 0 );
821 #ifdef BDB_TOOL_IDL_CACHING
823 bdb_tool_idl_cmp( const void *v1, const void *v2 )
825 const bdb_tool_idl_cache *c1 = v1, *c2 = v2;
828 if (( rc = c1->kstr.bv_len - c2->kstr.bv_len )) return rc;
829 return memcmp( c1->kstr.bv_val, c2->kstr.bv_val, c1->kstr.bv_len );
833 bdb_tool_idl_flush_one( void *v1, void *arg )
835 bdb_tool_idl_cache *ic = v1;
837 struct bdb_info *bdb = bdb_tool_info;
838 bdb_tool_idl_cache_entry *ice;
844 /* Freshly allocated, ignore it */
845 if ( !ic->head && ic->count <= BDB_IDL_DB_SIZE ) {
849 rc = db->cursor( db, NULL, &curs, 0 );
856 bv2DBT( &ic->kstr, &key );
858 data.size = data.ulen = sizeof( ID );
859 data.flags = DB_DBT_USERMEM;
862 rc = curs->c_get( curs, &key, &data, DB_SET );
863 /* If key already exists and we're writing a range... */
864 if ( rc == 0 && ic->count > BDB_IDL_DB_SIZE ) {
865 /* If it's not currently a range, must delete old info */
868 while ( curs->c_get( curs, &key, &data, DB_NEXT_DUP ) == 0 )
869 curs->c_del( curs, 0 );
872 /* Store range marker */
873 curs->c_put( curs, &key, &data, DB_KEYFIRST );
877 rc = curs->c_get( curs, &key, &data, DB_NEXT_DUP );
880 rc = curs->c_get( curs, &key, &data, DB_NEXT_DUP );
883 curs->c_del( curs, 0 );
885 BDB_ID2DISK( ic->last, &nid );
886 curs->c_put( curs, &key, &data, DB_KEYLAST );
888 } else if ( rc && rc != DB_NOTFOUND ) {
890 } else if ( ic->count > BDB_IDL_DB_SIZE ) {
891 /* range, didn't exist before */
893 rc = curs->c_put( curs, &key, &data, DB_KEYLAST );
895 BDB_ID2DISK( ic->first, &nid );
896 rc = curs->c_put( curs, &key, &data, DB_KEYLAST );
898 BDB_ID2DISK( ic->last, &nid );
899 rc = curs->c_put( curs, &key, &data, DB_KEYLAST );
908 /* Just a normal write */
910 for ( ice = ic->head, n=0; ice; ice = ice->next, n++ ) {
915 end = ic->count & (IDBLOCK-1);
919 for ( i=0; i<end; i++ ) {
920 if ( !ice->ids[i] ) continue;
921 BDB_ID2DISK( ice->ids[i], &nid );
922 rc = curs->c_put( curs, &key, &data, DB_NODUPDATA );
924 if ( rc == DB_KEYEXIST ) {
938 ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
939 ic->tail->next = bdb_tool_idl_free_list;
940 bdb_tool_idl_free_list = ic->head;
941 bdb->bi_idl_cache_size -= n;
942 ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
945 if ( ic != db->app_private ) {
948 ic->head = ic->tail = NULL;
950 curs->c_close( curs );
955 bdb_tool_idl_flush_db( DB *db, bdb_tool_idl_cache *ic )
957 Avlnode *root = db->app_private;
960 db->app_private = ic;
961 rc = avl_apply( root, bdb_tool_idl_flush_one, db, -1, AVL_INORDER );
962 avl_free( root, NULL );
963 db->app_private = NULL;
970 bdb_tool_idl_flush( BackendDB *be )
972 struct bdb_info *bdb = (struct bdb_info *) be->be_private;
977 for ( i=BDB_NDB; i < bdb->bi_ndatabases; i++ ) {
978 db = bdb->bi_databases[i]->bdi_db;
979 if ( !db->app_private ) continue;
980 rc = bdb_tool_idl_flush_db( db, NULL );
985 bdb->bi_idl_cache_size = 0;
990 int bdb_tool_idl_add(
997 struct bdb_info *bdb = (struct bdb_info *) be->be_private;
998 bdb_tool_idl_cache *ic, itmp;
999 bdb_tool_idl_cache_entry *ice;
1002 if ( !bdb->bi_idl_cache_max_size )
1003 return bdb_idl_insert_key( be, db, txn, key, id );
1005 DBT2bv( key, &itmp.kstr );
1007 ic = avl_find( (Avlnode *)db->app_private, &itmp, bdb_tool_idl_cmp );
1009 /* No entry yet, create one */
1016 ic = ch_malloc( sizeof( bdb_tool_idl_cache ) + itmp.kstr.bv_len );
1017 ic->kstr.bv_len = itmp.kstr.bv_len;
1018 ic->kstr.bv_val = (char *)(ic+1);
1019 AC_MEMCPY( ic->kstr.bv_val, itmp.kstr.bv_val, ic->kstr.bv_len );
1020 ic->head = ic->tail = NULL;
1023 avl_insert( (Avlnode **)&db->app_private, ic, bdb_tool_idl_cmp,
1026 /* load existing key count here */
1027 rc = db->cursor( db, NULL, &curs, 0 );
1028 if ( rc ) return rc;
1030 data.ulen = sizeof( ID );
1031 data.flags = DB_DBT_USERMEM;
1033 rc = curs->c_get( curs, key, &data, DB_SET );
1036 ic->count = BDB_IDL_DB_SIZE+1;
1040 curs->c_count( curs, &count, 0 );
1042 BDB_DISK2ID( &nid, &ic->first );
1045 curs->c_close( curs );
1047 /* are we a range already? */
1048 if ( ic->count > BDB_IDL_DB_SIZE ) {
1051 /* Are we at the limit, and converting to a range? */
1052 } else if ( ic->count == BDB_IDL_DB_SIZE ) {
1054 for ( ice = ic->head, n=0; ice; ice = ice->next, n++ )
1057 ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
1058 ic->tail->next = bdb_tool_idl_free_list;
1059 bdb_tool_idl_free_list = ic->head;
1060 bdb->bi_idl_cache_size -= n;
1061 ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
1063 ic->head = ic->tail = NULL;
1068 /* No free block, create that too */
1069 if ( !ic->tail || ( ic->count & (IDBLOCK-1)) == 0) {
1071 ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
1072 if ( bdb->bi_idl_cache_size >= bdb->bi_idl_cache_max_size ) {
1073 ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
1074 rc = bdb_tool_idl_flush_db( db, ic );
1077 avl_insert( (Avlnode **)&db->app_private, ic, bdb_tool_idl_cmp,
1079 ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
1081 bdb->bi_idl_cache_size++;
1082 if ( bdb_tool_idl_free_list ) {
1083 ice = bdb_tool_idl_free_list;
1084 bdb_tool_idl_free_list = ice->next;
1086 ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
1088 ice = ch_malloc( sizeof( bdb_tool_idl_cache_entry ));
1090 memset( ice, 0, sizeof( *ice ));
1094 ic->tail->next = ice;
1101 ice->ids[ ic->count & (IDBLOCK-1) ] = id;
1109 bdb_tool_trickle_task( void *ctx, void *ptr )
1114 ldap_pvt_thread_mutex_lock( &bdb_tool_trickle_mutex );
1116 ldap_pvt_thread_cond_wait( &bdb_tool_trickle_cond,
1117 &bdb_tool_trickle_mutex );
1118 if ( slapd_shutdown )
1120 env->memp_trickle( env, 30, &wrote );
1122 ldap_pvt_thread_mutex_unlock( &bdb_tool_trickle_mutex );
1128 bdb_tool_index_task( void *ctx, void *ptr )
1130 int base = *(int *)ptr;
1134 ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex );
1135 bdb_tool_index_tcount--;
1136 if ( !bdb_tool_index_tcount )
1137 ldap_pvt_thread_cond_signal( &bdb_tool_index_cond_main );
1138 ldap_pvt_thread_cond_wait( &bdb_tool_index_cond_work,
1139 &bdb_tool_index_mutex );
1140 ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex );
1141 if ( slapd_shutdown )
1144 bdb_tool_index_threads[base] = bdb_index_recrun( bdb_tool_ix_op,
1145 bdb_tool_info, bdb_tool_index_rec, bdb_tool_ix_id, base );