1 /* tools.c - tools for slap tools */
3 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
5 * Copyright 2000-2010 The OpenLDAP Foundation.
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted only as authorized by the OpenLDAP
12 * A copy of this license is available in the file LICENSE in the
13 * top-level directory of the distribution or, alternatively, at
14 * <http://www.OpenLDAP.org/license.html>.
20 #include <ac/string.h>
27 static DBC *cursor = NULL;
29 static EntryHeader eh;
30 static ID nid, previd = NOID;
31 static char ehbuf[16];
33 typedef struct dn_id {
38 #define HOLE_SIZE 4096
39 static dn_id hbuf[HOLE_SIZE], *holes = hbuf;
40 static unsigned nhmax = HOLE_SIZE;
41 static unsigned nholes;
43 static int index_nattrs;
45 #ifdef BDB_TOOL_IDL_CACHING
46 #define bdb_tool_idl_cmp BDB_SYMBOL(tool_idl_cmp)
47 #define bdb_tool_idl_flush_one BDB_SYMBOL(tool_idl_flush_one)
48 #define bdb_tool_idl_flush BDB_SYMBOL(tool_idl_flush)
50 static int bdb_tool_idl_flush( BackendDB *be );
54 typedef struct bdb_tool_idl_cache_entry {
55 struct bdb_tool_idl_cache_entry *next;
57 } bdb_tool_idl_cache_entry;
59 typedef struct bdb_tool_idl_cache {
61 bdb_tool_idl_cache_entry *head, *tail;
66 static bdb_tool_idl_cache_entry *bdb_tool_idl_free_list;
67 #endif /* BDB_TOOL_IDL_CACHING */
69 static ID bdb_tool_ix_id;
70 static Operation *bdb_tool_ix_op;
71 static int *bdb_tool_index_threads, bdb_tool_index_tcount;
72 static void *bdb_tool_index_rec;
73 static struct bdb_info *bdb_tool_info;
74 static ldap_pvt_thread_mutex_t bdb_tool_index_mutex;
75 static ldap_pvt_thread_cond_t bdb_tool_index_cond_main;
76 static ldap_pvt_thread_cond_t bdb_tool_index_cond_work;
78 #if DB_VERSION_FULL >= 0x04060000
81 /* Seems to slow things down too much in BDB 4.5 */
86 static ldap_pvt_thread_mutex_t bdb_tool_trickle_mutex;
87 static ldap_pvt_thread_cond_t bdb_tool_trickle_cond;
89 static void * bdb_tool_trickle_task( void *ctx, void *ptr );
92 static void * bdb_tool_index_task( void *ctx, void *ptr );
94 int bdb_tool_entry_open(
95 BackendDB *be, int mode )
97 struct bdb_info *bdb = (struct bdb_info *) be->be_private;
99 /* initialize key and data thangs */
102 key.flags = DB_DBT_USERMEM;
104 key.size = key.ulen = sizeof( nid );
105 data.flags = DB_DBT_USERMEM;
107 if (cursor == NULL) {
108 int rc = bdb->bi_id2entry->bdi_db->cursor(
109 bdb->bi_id2entry->bdi_db, bdb->bi_cache.c_txn, &cursor,
110 bdb->bi_db_opflags );
116 /* Set up for threaded slapindex */
117 if (( slapMode & (SLAP_TOOL_QUICK|SLAP_TOOL_READONLY)) == SLAP_TOOL_QUICK ) {
118 if ( !bdb_tool_info ) {
120 ldap_pvt_thread_mutex_init( &bdb_tool_trickle_mutex );
121 ldap_pvt_thread_cond_init( &bdb_tool_trickle_cond );
122 ldap_pvt_thread_pool_submit( &connection_pool, bdb_tool_trickle_task, bdb->bi_dbenv );
125 ldap_pvt_thread_mutex_init( &bdb_tool_index_mutex );
126 ldap_pvt_thread_cond_init( &bdb_tool_index_cond_main );
127 ldap_pvt_thread_cond_init( &bdb_tool_index_cond_work );
128 if ( bdb->bi_nattrs ) {
130 bdb_tool_index_threads = ch_malloc( slap_tool_thread_max * sizeof( int ));
131 bdb_tool_index_rec = ch_malloc( bdb->bi_nattrs * sizeof( IndexRec ));
132 bdb_tool_index_tcount = slap_tool_thread_max - 1;
133 for (i=1; i<slap_tool_thread_max; i++) {
134 int *ptr = ch_malloc( sizeof( int ));
136 ldap_pvt_thread_pool_submit( &connection_pool,
137 bdb_tool_index_task, ptr );
147 int bdb_tool_entry_close(
150 if ( bdb_tool_info ) {
153 ldap_pvt_thread_mutex_lock( &bdb_tool_trickle_mutex );
154 ldap_pvt_thread_cond_signal( &bdb_tool_trickle_cond );
155 ldap_pvt_thread_mutex_unlock( &bdb_tool_trickle_mutex );
157 ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex );
158 bdb_tool_index_tcount = slap_tool_thread_max - 1;
159 ldap_pvt_thread_cond_broadcast( &bdb_tool_index_cond_work );
160 ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex );
164 ch_free( eh.bv.bv_val );
169 cursor->c_close( cursor );
173 #ifdef BDB_TOOL_IDL_CACHING
174 bdb_tool_idl_flush( be );
179 fprintf( stderr, "Error, entries missing!\n");
180 for (i=0; i<nholes; i++) {
181 fprintf(stderr, " entry %ld: %s\n",
182 holes[i].id, holes[i].dn.bv_val);
190 ID bdb_tool_entry_next(
195 struct bdb_info *bdb = (struct bdb_info *) be->be_private;
197 assert( be != NULL );
198 assert( slapMode & SLAP_TOOL_MODE );
199 assert( bdb != NULL );
202 data.ulen = data.dlen = sizeof( ehbuf );
204 data.flags |= DB_DBT_PARTIAL;
205 rc = cursor->c_get( cursor, &key, &data, DB_NEXT );
208 /* If we're doing linear indexing and there are more attrs to
209 * index, and we're at the end of the database, start over.
211 if ( index_nattrs && rc == DB_NOTFOUND ) {
212 /* optional - do a checkpoint here? */
213 bdb_attr_info_free( bdb->bi_attrs[0] );
214 bdb->bi_attrs[0] = bdb->bi_attrs[index_nattrs];
216 rc = cursor->c_get( cursor, &key, &data, DB_FIRST );
225 BDB_DISK2ID( key.data, &id );
230 ID bdb_tool_dn2id_get(
237 EntryInfo *ei = NULL;
240 if ( BER_BVISEMPTY(dn) )
245 op.o_tmpmemctx = NULL;
246 op.o_tmpmfuncs = &ch_mfuncs;
248 rc = bdb_cache_find_ndn( &op, 0, dn, &ei );
249 if ( ei ) bdb_cache_entryinfo_unlock( ei );
250 if ( rc == DB_NOTFOUND )
256 Entry* bdb_tool_entry_get( BackendDB *be, ID id )
262 assert( be != NULL );
263 assert( slapMode & SLAP_TOOL_MODE );
265 if ( id != previd ) {
266 data.ulen = data.dlen = sizeof( ehbuf );
268 data.flags |= DB_DBT_PARTIAL;
270 BDB_ID2DISK( id, &nid );
271 rc = cursor->c_get( cursor, &key, &data, DB_SET );
277 eh.bv.bv_val = ehbuf;
278 eh.bv.bv_len = data.size;
279 rc = entry_header( &eh );
280 eoff = eh.data - eh.bv.bv_val;
285 data.flags &= ~DB_DBT_PARTIAL;
287 rc = cursor->c_get( cursor, &key, &data, DB_CURRENT );
288 if ( rc != DB_BUFFER_SMALL ) goto done;
290 /* Allocate a block and retrieve the data */
291 eh.bv.bv_len = eh.nvals * sizeof( struct berval ) + data.size;
292 eh.bv.bv_val = ch_realloc( eh.bv.bv_val, eh.bv.bv_len );
293 eh.data = eh.bv.bv_val + eh.nvals * sizeof( struct berval );
295 data.ulen = data.size;
297 /* Skip past already parsed nattr/nvals */
300 rc = cursor->c_get( cursor, &key, &data, DB_CURRENT );
303 #ifdef SLAP_ZONE_ALLOC
304 /* FIXME: will add ctx later */
305 rc = entry_decode( &eh, &e, NULL );
307 rc = entry_decode( &eh, &e );
310 if( rc == LDAP_SUCCESS ) {
313 if ( slapMode & SLAP_TOOL_READONLY ) {
314 struct bdb_info *bdb = (struct bdb_info *) be->be_private;
315 EntryInfo *ei = NULL;
321 op.o_tmpmemctx = NULL;
322 op.o_tmpmfuncs = &ch_mfuncs;
324 rc = bdb_cache_find_parent( &op, bdb->bi_cache.c_txn, id, &ei );
325 if ( rc == LDAP_SUCCESS ) {
326 bdb_cache_entryinfo_unlock( ei );
340 static int bdb_tool_next_id(
347 struct berval dn = e->e_name;
348 struct berval ndn = e->e_nname;
349 struct berval pdn, npdn;
350 EntryInfo *ei = NULL, eidummy;
353 if (ndn.bv_len == 0) {
358 rc = bdb_cache_find_ndn( op, tid, &ndn, &ei );
359 if ( ei ) bdb_cache_entryinfo_unlock( ei );
360 if ( rc == DB_NOTFOUND ) {
361 if ( !be_issuffix( op->o_bd, &ndn ) ) {
363 dnParent( &dn, &pdn );
364 dnParent( &ndn, &npdn );
367 rc = bdb_tool_next_id( op, tid, e, text, 1 );
373 /* If parent didn't exist, it was created just now
374 * and its ID is now in e->e_id. Make sure the current
375 * entry gets added under the new parent ID.
377 if ( eid != e->e_id ) {
378 eidummy.bei_id = e->e_id;
382 rc = bdb_next_id( op->o_bd, &e->e_id );
384 snprintf( text->bv_val, text->bv_len,
385 "next_id failed: %s (%d)",
386 db_strerror(rc), rc );
387 Debug( LDAP_DEBUG_ANY,
388 "=> bdb_tool_next_id: %s\n", text->bv_val, 0, 0 );
391 rc = bdb_dn2id_add( op, tid, ei, e );
393 snprintf( text->bv_val, text->bv_len,
394 "dn2id_add failed: %s (%d)",
395 db_strerror(rc), rc );
396 Debug( LDAP_DEBUG_ANY,
397 "=> bdb_tool_next_id: %s\n", text->bv_val, 0, 0 );
399 if ( nholes == nhmax - 1 ) {
400 if ( holes == hbuf ) {
401 holes = ch_malloc( nhmax * sizeof(dn_id) * 2 );
402 AC_MEMCPY( holes, hbuf, sizeof(hbuf) );
404 holes = ch_realloc( holes, nhmax * sizeof(dn_id) * 2 );
408 ber_dupbv( &holes[nholes].dn, &ndn );
409 holes[nholes++].id = e->e_id;
411 } else if ( !hole ) {
414 e->e_id = ei->bei_id;
416 for ( i=0; i<nholes; i++) {
417 if ( holes[i].id == e->e_id ) {
418 free(holes[i].dn.bv_val);
419 for (j=i;j<nholes;j++) holes[j] = holes[j+1];
423 } else if ( holes[i].id > e->e_id ) {
437 struct bdb_info *bdb = (struct bdb_info *) op->o_bd->be_private;
439 if ( !bdb->bi_nattrs )
442 if ( slapMode & SLAP_TOOL_QUICK ) {
447 ir = bdb_tool_index_rec;
448 memset(ir, 0, bdb->bi_nattrs * sizeof( IndexRec ));
450 for ( a = e->e_attrs; a != NULL; a = a->a_next ) {
451 rc = bdb_index_recset( bdb, a, a->a_desc->ad_type,
452 &a->a_desc->ad_tags, ir );
456 bdb_tool_ix_id = e->e_id;
458 ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex );
459 /* Wait for all threads to be ready */
460 while ( bdb_tool_index_tcount ) {
461 ldap_pvt_thread_cond_wait( &bdb_tool_index_cond_main,
462 &bdb_tool_index_mutex );
464 for ( i=1; i<slap_tool_thread_max; i++ )
465 bdb_tool_index_threads[i] = LDAP_BUSY;
466 bdb_tool_index_tcount = slap_tool_thread_max - 1;
467 ldap_pvt_thread_cond_broadcast( &bdb_tool_index_cond_work );
468 ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex );
469 rc = bdb_index_recrun( op, bdb, ir, e->e_id, 0 );
472 ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex );
473 for ( i=1; i<slap_tool_thread_max; i++ ) {
474 if ( bdb_tool_index_threads[i] == LDAP_BUSY ) {
475 ldap_pvt_thread_cond_wait( &bdb_tool_index_cond_main,
476 &bdb_tool_index_mutex );
480 if ( bdb_tool_index_threads[i] ) {
481 rc = bdb_tool_index_threads[i];
485 ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex );
488 return bdb_index_entry_add( op, txn, e );
492 ID bdb_tool_entry_put(
495 struct berval *text )
498 struct bdb_info *bdb = (struct bdb_info *) be->be_private;
503 assert( be != NULL );
504 assert( slapMode & SLAP_TOOL_MODE );
506 assert( text != NULL );
507 assert( text->bv_val != NULL );
508 assert( text->bv_val[0] == '\0' ); /* overconservative? */
510 Debug( LDAP_DEBUG_TRACE, "=> " LDAP_XSTRING(bdb_tool_entry_put)
511 "( %ld, \"%s\" )\n", (long) e->e_id, e->e_dn, 0 );
513 if (! (slapMode & SLAP_TOOL_QUICK)) {
514 rc = TXN_BEGIN( bdb->bi_dbenv, NULL, &tid,
515 bdb->bi_db_opflags );
517 snprintf( text->bv_val, text->bv_len,
518 "txn_begin failed: %s (%d)",
519 db_strerror(rc), rc );
520 Debug( LDAP_DEBUG_ANY,
521 "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
522 text->bv_val, 0, 0 );
529 op.o_tmpmemctx = NULL;
530 op.o_tmpmfuncs = &ch_mfuncs;
532 /* add dn2id indices */
533 rc = bdb_tool_next_id( &op, tid, e, text, 0 );
539 if (( slapMode & SLAP_TOOL_QUICK ) && (( e->e_id & 0xfff ) == 0xfff )) {
540 ldap_pvt_thread_cond_signal( &bdb_tool_trickle_cond );
544 if ( !bdb->bi_linear_index )
545 rc = bdb_tool_index_add( &op, tid, e );
547 snprintf( text->bv_val, text->bv_len,
548 "index_entry_add failed: %s (%d)",
549 rc == LDAP_OTHER ? "Internal error" :
550 db_strerror(rc), rc );
551 Debug( LDAP_DEBUG_ANY,
552 "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
553 text->bv_val, 0, 0 );
558 rc = bdb_id2entry_add( be, tid, e );
560 snprintf( text->bv_val, text->bv_len,
561 "id2entry_add failed: %s (%d)",
562 db_strerror(rc), rc );
563 Debug( LDAP_DEBUG_ANY,
564 "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
565 text->bv_val, 0, 0 );
571 if ( !( slapMode & SLAP_TOOL_QUICK )) {
572 rc = TXN_COMMIT( tid, 0 );
574 snprintf( text->bv_val, text->bv_len,
575 "txn_commit failed: %s (%d)",
576 db_strerror(rc), rc );
577 Debug( LDAP_DEBUG_ANY,
578 "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
579 text->bv_val, 0, 0 );
585 if ( !( slapMode & SLAP_TOOL_QUICK )) {
587 snprintf( text->bv_val, text->bv_len,
588 "txn_aborted! %s (%d)",
589 rc == LDAP_OTHER ? "Internal error" :
590 db_strerror(rc), rc );
591 Debug( LDAP_DEBUG_ANY,
592 "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
593 text->bv_val, 0, 0 );
601 int bdb_tool_entry_reindex(
604 AttributeDescription **adv )
606 struct bdb_info *bi = (struct bdb_info *) be->be_private;
613 Debug( LDAP_DEBUG_ARGS,
614 "=> " LDAP_XSTRING(bdb_tool_entry_reindex) "( %ld )\n",
617 /* No indexes configured, nothing to do. Could return an
618 * error here to shortcut things.
624 /* Check for explicit list of attrs to index */
628 if ( bi->bi_attrs[0]->ai_desc != adv[0] ) {
630 for ( n = 0; adv[n]; n++ ) ;
633 for ( i = 0; i < n; i++ ) {
634 AttributeDescription *ad = adv[i];
635 for ( j = i-1; j>=0; j--) {
636 if ( SLAP_PTRCMP( adv[j], ad ) <= 0 ) break;
643 for ( i = 0; adv[i]; i++ ) {
644 if ( bi->bi_attrs[i]->ai_desc != adv[i] ) {
645 for ( j = i+1; j < bi->bi_nattrs; j++ ) {
646 if ( bi->bi_attrs[j]->ai_desc == adv[i] ) {
647 AttrInfo *ai = bi->bi_attrs[i];
648 bi->bi_attrs[i] = bi->bi_attrs[j];
649 bi->bi_attrs[j] = ai;
653 if ( j == bi->bi_nattrs ) {
654 Debug( LDAP_DEBUG_ANY,
655 LDAP_XSTRING(bdb_tool_entry_reindex)
656 ": no index configured for %s\n",
657 adv[i]->ad_cname.bv_val, 0, 0 );
665 /* Get the first attribute to index */
666 if (bi->bi_linear_index && !index_nattrs) {
667 index_nattrs = bi->bi_nattrs - 1;
671 e = bdb_tool_entry_get( be, id );
674 Debug( LDAP_DEBUG_ANY,
675 LDAP_XSTRING(bdb_tool_entry_reindex)
676 ": could not locate id=%ld\n",
681 if (! (slapMode & SLAP_TOOL_QUICK)) {
682 rc = TXN_BEGIN( bi->bi_dbenv, NULL, &tid, bi->bi_db_opflags );
684 Debug( LDAP_DEBUG_ANY,
685 "=> " LDAP_XSTRING(bdb_tool_entry_reindex) ": "
686 "txn_begin failed: %s (%d)\n",
687 db_strerror(rc), rc, 0 );
693 * just (re)add them for now
694 * assume that some other routine (not yet implemented)
695 * will zap index databases
699 Debug( LDAP_DEBUG_TRACE,
700 "=> " LDAP_XSTRING(bdb_tool_entry_reindex) "( %ld, \"%s\" )\n",
701 (long) id, e->e_dn, 0 );
705 op.o_tmpmemctx = NULL;
706 op.o_tmpmfuncs = &ch_mfuncs;
708 rc = bdb_tool_index_add( &op, tid, e );
712 if (! (slapMode & SLAP_TOOL_QUICK)) {
713 rc = TXN_COMMIT( tid, 0 );
715 Debug( LDAP_DEBUG_ANY,
716 "=> " LDAP_XSTRING(bdb_tool_entry_reindex)
717 ": txn_commit failed: %s (%d)\n",
718 db_strerror(rc), rc, 0 );
724 if (! (slapMode & SLAP_TOOL_QUICK)) {
726 Debug( LDAP_DEBUG_ANY,
727 "=> " LDAP_XSTRING(bdb_tool_entry_reindex)
728 ": txn_aborted! %s (%d)\n",
729 db_strerror(rc), rc, 0 );
733 bdb_entry_release( &op, e, 0 );
738 ID bdb_tool_entry_modify(
741 struct berval *text )
744 struct bdb_info *bdb = (struct bdb_info *) be->be_private;
749 assert( be != NULL );
750 assert( slapMode & SLAP_TOOL_MODE );
752 assert( text != NULL );
753 assert( text->bv_val != NULL );
754 assert( text->bv_val[0] == '\0' ); /* overconservative? */
756 assert ( e->e_id != NOID );
758 Debug( LDAP_DEBUG_TRACE,
759 "=> " LDAP_XSTRING(bdb_tool_entry_modify) "( %ld, \"%s\" )\n",
760 (long) e->e_id, e->e_dn, 0 );
762 if (! (slapMode & SLAP_TOOL_QUICK)) {
764 cursor->c_close( cursor );
767 rc = TXN_BEGIN( bdb->bi_dbenv, NULL, &tid,
768 bdb->bi_db_opflags );
770 snprintf( text->bv_val, text->bv_len,
771 "txn_begin failed: %s (%d)",
772 db_strerror(rc), rc );
773 Debug( LDAP_DEBUG_ANY,
774 "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n",
775 text->bv_val, 0, 0 );
782 op.o_tmpmemctx = NULL;
783 op.o_tmpmfuncs = &ch_mfuncs;
786 rc = bdb_id2entry_update( be, tid, e );
788 snprintf( text->bv_val, text->bv_len,
789 "id2entry_add failed: %s (%d)",
790 db_strerror(rc), rc );
791 Debug( LDAP_DEBUG_ANY,
792 "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n",
793 text->bv_val, 0, 0 );
799 if (! (slapMode & SLAP_TOOL_QUICK)) {
800 rc = TXN_COMMIT( tid, 0 );
802 snprintf( text->bv_val, text->bv_len,
803 "txn_commit failed: %s (%d)",
804 db_strerror(rc), rc );
805 Debug( LDAP_DEBUG_ANY,
806 "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": "
807 "%s\n", text->bv_val, 0, 0 );
813 if (! (slapMode & SLAP_TOOL_QUICK)) {
815 snprintf( text->bv_val, text->bv_len,
816 "txn_aborted! %s (%d)",
817 db_strerror(rc), rc );
818 Debug( LDAP_DEBUG_ANY,
819 "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n",
820 text->bv_val, 0, 0 );
828 #ifdef BDB_TOOL_IDL_CACHING
830 bdb_tool_idl_cmp( const void *v1, const void *v2 )
832 const bdb_tool_idl_cache *c1 = v1, *c2 = v2;
835 if (( rc = c1->kstr.bv_len - c2->kstr.bv_len )) return rc;
836 return memcmp( c1->kstr.bv_val, c2->kstr.bv_val, c1->kstr.bv_len );
840 bdb_tool_idl_flush_one( void *v1, void *arg )
842 bdb_tool_idl_cache *ic = v1;
844 struct bdb_info *bdb = bdb_tool_info;
845 bdb_tool_idl_cache_entry *ice;
851 /* Freshly allocated, ignore it */
852 if ( !ic->head && ic->count <= BDB_IDL_DB_SIZE ) {
856 rc = db->cursor( db, NULL, &curs, 0 );
863 bv2DBT( &ic->kstr, &key );
865 data.size = data.ulen = sizeof( ID );
866 data.flags = DB_DBT_USERMEM;
869 rc = curs->c_get( curs, &key, &data, DB_SET );
870 /* If key already exists and we're writing a range... */
871 if ( rc == 0 && ic->count > BDB_IDL_DB_SIZE ) {
872 /* If it's not currently a range, must delete old info */
875 while ( curs->c_get( curs, &key, &data, DB_NEXT_DUP ) == 0 )
876 curs->c_del( curs, 0 );
879 /* Store range marker */
880 curs->c_put( curs, &key, &data, DB_KEYFIRST );
884 rc = curs->c_get( curs, &key, &data, DB_NEXT_DUP );
887 rc = curs->c_get( curs, &key, &data, DB_NEXT_DUP );
890 curs->c_del( curs, 0 );
892 BDB_ID2DISK( ic->last, &nid );
893 curs->c_put( curs, &key, &data, DB_KEYLAST );
895 } else if ( rc && rc != DB_NOTFOUND ) {
897 } else if ( ic->count > BDB_IDL_DB_SIZE ) {
898 /* range, didn't exist before */
900 rc = curs->c_put( curs, &key, &data, DB_KEYLAST );
902 BDB_ID2DISK( ic->first, &nid );
903 rc = curs->c_put( curs, &key, &data, DB_KEYLAST );
905 BDB_ID2DISK( ic->last, &nid );
906 rc = curs->c_put( curs, &key, &data, DB_KEYLAST );
915 /* Just a normal write */
917 for ( ice = ic->head, n=0; ice; ice = ice->next, n++ ) {
922 end = ic->count & (IDBLOCK-1);
926 for ( i=0; i<end; i++ ) {
927 if ( !ice->ids[i] ) continue;
928 BDB_ID2DISK( ice->ids[i], &nid );
929 rc = curs->c_put( curs, &key, &data, DB_NODUPDATA );
931 if ( rc == DB_KEYEXIST ) {
945 ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
946 ic->tail->next = bdb_tool_idl_free_list;
947 bdb_tool_idl_free_list = ic->head;
948 bdb->bi_idl_cache_size -= n;
949 ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
952 if ( ic != db->app_private ) {
955 ic->head = ic->tail = NULL;
957 curs->c_close( curs );
962 bdb_tool_idl_flush_db( DB *db, bdb_tool_idl_cache *ic )
964 Avlnode *root = db->app_private;
967 db->app_private = ic;
968 rc = avl_apply( root, bdb_tool_idl_flush_one, db, -1, AVL_INORDER );
969 avl_free( root, NULL );
970 db->app_private = NULL;
977 bdb_tool_idl_flush( BackendDB *be )
979 struct bdb_info *bdb = (struct bdb_info *) be->be_private;
984 for ( i=BDB_NDB; i < bdb->bi_ndatabases; i++ ) {
985 db = bdb->bi_databases[i]->bdi_db;
986 if ( !db->app_private ) continue;
987 rc = bdb_tool_idl_flush_db( db, NULL );
992 bdb->bi_idl_cache_size = 0;
997 int bdb_tool_idl_add(
1004 struct bdb_info *bdb = (struct bdb_info *) be->be_private;
1005 bdb_tool_idl_cache *ic, itmp;
1006 bdb_tool_idl_cache_entry *ice;
1009 if ( !bdb->bi_idl_cache_max_size )
1010 return bdb_idl_insert_key( be, db, txn, key, id );
1012 DBT2bv( key, &itmp.kstr );
1014 ic = avl_find( (Avlnode *)db->app_private, &itmp, bdb_tool_idl_cmp );
1016 /* No entry yet, create one */
1023 ic = ch_malloc( sizeof( bdb_tool_idl_cache ) + itmp.kstr.bv_len );
1024 ic->kstr.bv_len = itmp.kstr.bv_len;
1025 ic->kstr.bv_val = (char *)(ic+1);
1026 AC_MEMCPY( ic->kstr.bv_val, itmp.kstr.bv_val, ic->kstr.bv_len );
1027 ic->head = ic->tail = NULL;
1030 avl_insert( (Avlnode **)&db->app_private, ic, bdb_tool_idl_cmp,
1033 /* load existing key count here */
1034 rc = db->cursor( db, NULL, &curs, 0 );
1035 if ( rc ) return rc;
1037 data.ulen = sizeof( ID );
1038 data.flags = DB_DBT_USERMEM;
1040 rc = curs->c_get( curs, key, &data, DB_SET );
1043 ic->count = BDB_IDL_DB_SIZE+1;
1047 curs->c_count( curs, &count, 0 );
1049 BDB_DISK2ID( &nid, &ic->first );
1052 curs->c_close( curs );
1054 /* are we a range already? */
1055 if ( ic->count > BDB_IDL_DB_SIZE ) {
1058 /* Are we at the limit, and converting to a range? */
1059 } else if ( ic->count == BDB_IDL_DB_SIZE ) {
1061 for ( ice = ic->head, n=0; ice; ice = ice->next, n++ )
1064 ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
1065 ic->tail->next = bdb_tool_idl_free_list;
1066 bdb_tool_idl_free_list = ic->head;
1067 bdb->bi_idl_cache_size -= n;
1068 ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
1070 ic->head = ic->tail = NULL;
1075 /* No free block, create that too */
1076 if ( !ic->tail || ( ic->count & (IDBLOCK-1)) == 0) {
1078 ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
1079 if ( bdb->bi_idl_cache_size >= bdb->bi_idl_cache_max_size ) {
1080 ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
1081 rc = bdb_tool_idl_flush_db( db, ic );
1084 avl_insert( (Avlnode **)&db->app_private, ic, bdb_tool_idl_cmp,
1086 ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
1088 bdb->bi_idl_cache_size++;
1089 if ( bdb_tool_idl_free_list ) {
1090 ice = bdb_tool_idl_free_list;
1091 bdb_tool_idl_free_list = ice->next;
1093 ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
1095 ice = ch_malloc( sizeof( bdb_tool_idl_cache_entry ));
1097 memset( ice, 0, sizeof( *ice ));
1101 ic->tail->next = ice;
1108 ice->ids[ ic->count & (IDBLOCK-1) ] = id;
1117 bdb_tool_trickle_task( void *ctx, void *ptr )
1122 ldap_pvt_thread_mutex_lock( &bdb_tool_trickle_mutex );
1124 ldap_pvt_thread_cond_wait( &bdb_tool_trickle_cond,
1125 &bdb_tool_trickle_mutex );
1126 if ( slapd_shutdown )
1128 env->memp_trickle( env, 30, &wrote );
1130 ldap_pvt_thread_mutex_unlock( &bdb_tool_trickle_mutex );
1137 bdb_tool_index_task( void *ctx, void *ptr )
1139 int base = *(int *)ptr;
1143 ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex );
1144 bdb_tool_index_tcount--;
1145 if ( !bdb_tool_index_tcount )
1146 ldap_pvt_thread_cond_signal( &bdb_tool_index_cond_main );
1147 ldap_pvt_thread_cond_wait( &bdb_tool_index_cond_work,
1148 &bdb_tool_index_mutex );
1149 ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex );
1150 if ( slapd_shutdown )
1153 bdb_tool_index_threads[base] = bdb_index_recrun( bdb_tool_ix_op,
1154 bdb_tool_info, bdb_tool_index_rec, bdb_tool_ix_id, base );