1 /* tools.c - tools for slap tools */
3 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
5 * Copyright 2000-2011 The OpenLDAP Foundation.
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted only as authorized by the OpenLDAP
12 * A copy of this license is available in the file LICENSE in the
13 * top-level directory of the distribution or, alternatively, at
14 * <http://www.OpenLDAP.org/license.html>.
20 #include <ac/string.h>
27 static DBC *cursor = NULL;
29 static EntryHeader eh;
30 static ID nid, previd = NOID;
31 static char ehbuf[16];
33 typedef struct dn_id {
38 #define HOLE_SIZE 4096
39 static dn_id hbuf[HOLE_SIZE], *holes = hbuf;
40 static unsigned nhmax = HOLE_SIZE;
41 static unsigned nholes;
43 static int index_nattrs;
45 static struct berval *tool_base;
46 static int tool_scope;
47 static Filter *tool_filter;
48 static Entry *tool_next_entry;
50 #ifdef BDB_TOOL_IDL_CACHING
51 #define bdb_tool_idl_cmp BDB_SYMBOL(tool_idl_cmp)
52 #define bdb_tool_idl_flush_one BDB_SYMBOL(tool_idl_flush_one)
53 #define bdb_tool_idl_flush BDB_SYMBOL(tool_idl_flush)
55 static int bdb_tool_idl_flush( BackendDB *be );
59 typedef struct bdb_tool_idl_cache_entry {
60 struct bdb_tool_idl_cache_entry *next;
62 } bdb_tool_idl_cache_entry;
64 typedef struct bdb_tool_idl_cache {
66 bdb_tool_idl_cache_entry *head, *tail;
71 static bdb_tool_idl_cache_entry *bdb_tool_idl_free_list;
72 #endif /* BDB_TOOL_IDL_CACHING */
74 static ID bdb_tool_ix_id;
75 static Operation *bdb_tool_ix_op;
76 static int *bdb_tool_index_threads, bdb_tool_index_tcount;
77 static void *bdb_tool_index_rec;
78 static struct bdb_info *bdb_tool_info;
79 static ldap_pvt_thread_mutex_t bdb_tool_index_mutex;
80 static ldap_pvt_thread_cond_t bdb_tool_index_cond_main;
81 static ldap_pvt_thread_cond_t bdb_tool_index_cond_work;
83 #if DB_VERSION_FULL >= 0x04060000
86 /* Seems to slow things down too much in BDB 4.5 */
91 static ldap_pvt_thread_mutex_t bdb_tool_trickle_mutex;
92 static ldap_pvt_thread_cond_t bdb_tool_trickle_cond;
93 static ldap_pvt_thread_cond_t bdb_tool_trickle_cond_end;
95 static void * bdb_tool_trickle_task( void *ctx, void *ptr );
96 static int bdb_tool_trickle_active;
99 static void * bdb_tool_index_task( void *ctx, void *ptr );
102 bdb_tool_entry_get_int( BackendDB *be, ID id, Entry **ep );
104 static int bdb_tool_threads;
106 int bdb_tool_entry_open(
107 BackendDB *be, int mode )
109 struct bdb_info *bdb = (struct bdb_info *) be->be_private;
111 /* initialize key and data thangs */
114 key.flags = DB_DBT_USERMEM;
116 key.size = key.ulen = sizeof( nid );
117 data.flags = DB_DBT_USERMEM;
119 if (cursor == NULL) {
120 int rc = bdb->bi_id2entry->bdi_db->cursor(
121 bdb->bi_id2entry->bdi_db, bdb->bi_cache.c_txn, &cursor,
122 bdb->bi_db_opflags );
128 /* Set up for threaded slapindex */
129 if (( slapMode & (SLAP_TOOL_QUICK|SLAP_TOOL_READONLY)) == SLAP_TOOL_QUICK ) {
130 if ( !bdb_tool_info ) {
132 ldap_pvt_thread_mutex_init( &bdb_tool_trickle_mutex );
133 ldap_pvt_thread_cond_init( &bdb_tool_trickle_cond );
134 ldap_pvt_thread_cond_init( &bdb_tool_trickle_cond_end );
135 ldap_pvt_thread_pool_submit( &connection_pool, bdb_tool_trickle_task, bdb->bi_dbenv );
138 ldap_pvt_thread_mutex_init( &bdb_tool_index_mutex );
139 ldap_pvt_thread_cond_init( &bdb_tool_index_cond_main );
140 ldap_pvt_thread_cond_init( &bdb_tool_index_cond_work );
141 if ( bdb->bi_nattrs ) {
143 bdb_tool_threads = slap_tool_thread_max - 1;
144 bdb_tool_index_threads = ch_malloc( bdb_tool_threads * sizeof( int ));
145 bdb_tool_index_rec = ch_malloc( bdb->bi_nattrs * sizeof( IndexRec ));
146 bdb_tool_index_tcount = bdb_tool_threads - 1;
147 for (i=1; i<bdb_tool_threads; i++) {
148 int *ptr = ch_malloc( sizeof( int ));
150 ldap_pvt_thread_pool_submit( &connection_pool,
151 bdb_tool_index_task, ptr );
161 int bdb_tool_entry_close(
164 if ( bdb_tool_info ) {
167 ldap_pvt_thread_mutex_lock( &bdb_tool_trickle_mutex );
169 /* trickle thread may not have started yet */
170 while ( !bdb_tool_trickle_active )
171 ldap_pvt_thread_cond_wait( &bdb_tool_trickle_cond_end,
172 &bdb_tool_trickle_mutex );
174 ldap_pvt_thread_cond_signal( &bdb_tool_trickle_cond );
175 while ( bdb_tool_trickle_active )
176 ldap_pvt_thread_cond_wait( &bdb_tool_trickle_cond_end,
177 &bdb_tool_trickle_mutex );
178 ldap_pvt_thread_mutex_unlock( &bdb_tool_trickle_mutex );
180 ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex );
182 /* There might still be some threads starting */
183 while ( bdb_tool_index_tcount ) {
184 ldap_pvt_thread_cond_wait( &bdb_tool_index_cond_main,
185 &bdb_tool_index_mutex );
188 bdb_tool_index_tcount = bdb_tool_threads - 1;
189 ldap_pvt_thread_cond_broadcast( &bdb_tool_index_cond_work );
191 /* Make sure all threads are stopped */
192 while ( bdb_tool_index_tcount ) {
193 ldap_pvt_thread_cond_wait( &bdb_tool_index_cond_main,
194 &bdb_tool_index_mutex );
196 ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex );
198 bdb_tool_info = NULL;
200 ch_free( bdb_tool_index_threads );
201 ch_free( bdb_tool_index_rec );
202 bdb_tool_index_tcount = bdb_tool_threads - 1;
206 ch_free( eh.bv.bv_val );
211 cursor->c_close( cursor );
215 #ifdef BDB_TOOL_IDL_CACHING
216 bdb_tool_idl_flush( be );
221 fprintf( stderr, "Error, entries missing!\n");
222 for (i=0; i<nholes; i++) {
223 fprintf(stderr, " entry %ld: %s\n",
224 holes[i].id, holes[i].dn.bv_val);
233 bdb_tool_entry_first_x(
243 return bdb_tool_entry_next( be );
246 ID bdb_tool_entry_next(
251 struct bdb_info *bdb;
253 assert( be != NULL );
254 assert( slapMode & SLAP_TOOL_MODE );
256 bdb = (struct bdb_info *) be->be_private;
257 assert( bdb != NULL );
261 data.ulen = data.dlen = sizeof( ehbuf );
263 data.flags |= DB_DBT_PARTIAL;
264 rc = cursor->c_get( cursor, &key, &data, DB_NEXT );
267 /* If we're doing linear indexing and there are more attrs to
268 * index, and we're at the end of the database, start over.
270 if ( index_nattrs && rc == DB_NOTFOUND ) {
271 /* optional - do a checkpoint here? */
272 bdb_attr_info_free( bdb->bi_attrs[0] );
273 bdb->bi_attrs[0] = bdb->bi_attrs[index_nattrs];
275 rc = cursor->c_get( cursor, &key, &data, DB_FIRST );
284 BDB_DISK2ID( key.data, &id );
287 if ( tool_filter || tool_base ) {
288 static Operation op = {0};
289 static Opheader ohdr = {0};
293 op.o_tmpmemctx = NULL;
294 op.o_tmpmfuncs = &ch_mfuncs;
296 if ( tool_next_entry ) {
297 bdb_entry_release( &op, tool_next_entry, 0 );
298 tool_next_entry = NULL;
301 rc = bdb_tool_entry_get_int( be, id, &tool_next_entry );
302 if ( rc == LDAP_NO_SUCH_OBJECT ) {
306 assert( tool_next_entry != NULL );
309 /* TODO: needed until BDB_HIER is handled accordingly
310 * in bdb_tool_entry_get_int() */
311 if ( tool_base && !dnIsSuffixScope( &tool_next_entry->e_nname, tool_base, tool_scope ) )
313 bdb_entry_release( &op, tool_next_entry, 0 );
314 tool_next_entry = NULL;
319 if ( tool_filter && test_filter( NULL, tool_next_entry, tool_filter ) != LDAP_COMPARE_TRUE )
321 bdb_entry_release( &op, tool_next_entry, 0 );
322 tool_next_entry = NULL;
330 ID bdb_tool_dn2id_get(
337 EntryInfo *ei = NULL;
340 if ( BER_BVISEMPTY(dn) )
345 op.o_tmpmemctx = NULL;
346 op.o_tmpmfuncs = &ch_mfuncs;
348 rc = bdb_cache_find_ndn( &op, 0, dn, &ei );
349 if ( ei ) bdb_cache_entryinfo_unlock( ei );
350 if ( rc == DB_NOTFOUND )
357 bdb_tool_entry_get_int( BackendDB *be, ID id, Entry **ep )
363 assert( be != NULL );
364 assert( slapMode & SLAP_TOOL_MODE );
366 if ( ( tool_filter || tool_base ) && id == previd && tool_next_entry != NULL ) {
367 *ep = tool_next_entry;
368 tool_next_entry = NULL;
372 if ( id != previd ) {
373 data.ulen = data.dlen = sizeof( ehbuf );
375 data.flags |= DB_DBT_PARTIAL;
377 BDB_ID2DISK( id, &nid );
378 rc = cursor->c_get( cursor, &key, &data, DB_SET );
387 eh.bv.bv_val = ehbuf;
388 eh.bv.bv_len = data.size;
389 rc = entry_header( &eh );
390 eoff = eh.data - eh.bv.bv_val;
398 data.flags &= ~DB_DBT_PARTIAL;
400 rc = cursor->c_get( cursor, &key, &data, DB_CURRENT );
401 if ( rc != DB_BUFFER_SMALL ) {
406 /* Allocate a block and retrieve the data */
407 eh.bv.bv_len = eh.nvals * sizeof( struct berval ) + data.size;
408 eh.bv.bv_val = ch_realloc( eh.bv.bv_val, eh.bv.bv_len );
409 eh.data = eh.bv.bv_val + eh.nvals * sizeof( struct berval );
411 data.ulen = data.size;
413 /* Skip past already parsed nattr/nvals */
416 rc = cursor->c_get( cursor, &key, &data, DB_CURRENT );
423 /* TODO: handle BDB_HIER accordingly */
424 if ( tool_base != NULL ) {
426 entry_decode_dn( &eh, NULL, &ndn );
428 if ( !dnIsSuffixScope( &ndn, tool_base, tool_scope ) ) {
429 return LDAP_NO_SUCH_OBJECT;
434 #ifdef SLAP_ZONE_ALLOC
435 /* FIXME: will add ctx later */
436 rc = entry_decode( &eh, &e, NULL );
438 rc = entry_decode( &eh, &e );
441 if( rc == LDAP_SUCCESS ) {
444 if ( slapMode & SLAP_TOOL_READONLY ) {
445 struct bdb_info *bdb = (struct bdb_info *) be->be_private;
446 EntryInfo *ei = NULL;
452 op.o_tmpmemctx = NULL;
453 op.o_tmpmfuncs = &ch_mfuncs;
455 rc = bdb_cache_find_parent( &op, bdb->bi_cache.c_txn, id, &ei );
456 if ( rc == LDAP_SUCCESS ) {
457 bdb_cache_entryinfo_unlock( ei );
476 bdb_tool_entry_get( BackendDB *be, ID id )
480 (void)bdb_tool_entry_get_int( be, id, &e );
484 static int bdb_tool_next_id(
491 struct berval dn = e->e_name;
492 struct berval ndn = e->e_nname;
493 struct berval pdn, npdn;
494 EntryInfo *ei = NULL, eidummy;
497 if (ndn.bv_len == 0) {
502 rc = bdb_cache_find_ndn( op, tid, &ndn, &ei );
503 if ( ei ) bdb_cache_entryinfo_unlock( ei );
504 if ( rc == DB_NOTFOUND ) {
505 if ( !be_issuffix( op->o_bd, &ndn ) ) {
507 dnParent( &dn, &pdn );
508 dnParent( &ndn, &npdn );
511 rc = bdb_tool_next_id( op, tid, e, text, 1 );
517 /* If parent didn't exist, it was created just now
518 * and its ID is now in e->e_id. Make sure the current
519 * entry gets added under the new parent ID.
521 if ( eid != e->e_id ) {
522 eidummy.bei_id = e->e_id;
526 rc = bdb_next_id( op->o_bd, &e->e_id );
528 snprintf( text->bv_val, text->bv_len,
529 "next_id failed: %s (%d)",
530 db_strerror(rc), rc );
531 Debug( LDAP_DEBUG_ANY,
532 "=> bdb_tool_next_id: %s\n", text->bv_val, 0, 0 );
535 rc = bdb_dn2id_add( op, tid, ei, e );
537 snprintf( text->bv_val, text->bv_len,
538 "dn2id_add failed: %s (%d)",
539 db_strerror(rc), rc );
540 Debug( LDAP_DEBUG_ANY,
541 "=> bdb_tool_next_id: %s\n", text->bv_val, 0, 0 );
543 if ( nholes == nhmax - 1 ) {
544 if ( holes == hbuf ) {
545 holes = ch_malloc( nhmax * sizeof(dn_id) * 2 );
546 AC_MEMCPY( holes, hbuf, sizeof(hbuf) );
548 holes = ch_realloc( holes, nhmax * sizeof(dn_id) * 2 );
552 ber_dupbv( &holes[nholes].dn, &ndn );
553 holes[nholes++].id = e->e_id;
555 } else if ( !hole ) {
558 e->e_id = ei->bei_id;
560 for ( i=0; i<nholes; i++) {
561 if ( holes[i].id == e->e_id ) {
562 free(holes[i].dn.bv_val);
563 for (j=i;j<nholes;j++) holes[j] = holes[j+1];
567 } else if ( holes[i].id > e->e_id ) {
581 struct bdb_info *bdb = (struct bdb_info *) op->o_bd->be_private;
583 if ( !bdb->bi_nattrs )
586 if ( slapMode & SLAP_TOOL_QUICK ) {
591 ir = bdb_tool_index_rec;
592 memset(ir, 0, bdb->bi_nattrs * sizeof( IndexRec ));
594 for ( a = e->e_attrs; a != NULL; a = a->a_next ) {
595 rc = bdb_index_recset( bdb, a, a->a_desc->ad_type,
596 &a->a_desc->ad_tags, ir );
600 bdb_tool_ix_id = e->e_id;
602 ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex );
603 /* Wait for all threads to be ready */
604 while ( bdb_tool_index_tcount ) {
605 ldap_pvt_thread_cond_wait( &bdb_tool_index_cond_main,
606 &bdb_tool_index_mutex );
608 for ( i=1; i<bdb_tool_threads; i++ )
609 bdb_tool_index_threads[i] = LDAP_BUSY;
610 bdb_tool_index_tcount = bdb_tool_threads - 1;
611 ldap_pvt_thread_cond_broadcast( &bdb_tool_index_cond_work );
612 ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex );
613 rc = bdb_index_recrun( op, bdb, ir, e->e_id, 0 );
616 ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex );
617 for ( i=1; i<bdb_tool_threads; i++ ) {
618 if ( bdb_tool_index_threads[i] == LDAP_BUSY ) {
619 ldap_pvt_thread_cond_wait( &bdb_tool_index_cond_main,
620 &bdb_tool_index_mutex );
624 if ( bdb_tool_index_threads[i] ) {
625 rc = bdb_tool_index_threads[i];
629 ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex );
632 return bdb_index_entry_add( op, txn, e );
636 ID bdb_tool_entry_put(
639 struct berval *text )
642 struct bdb_info *bdb;
647 assert( be != NULL );
648 assert( slapMode & SLAP_TOOL_MODE );
650 assert( text != NULL );
651 assert( text->bv_val != NULL );
652 assert( text->bv_val[0] == '\0' ); /* overconservative? */
654 Debug( LDAP_DEBUG_TRACE, "=> " LDAP_XSTRING(bdb_tool_entry_put)
655 "( %ld, \"%s\" )\n", (long) e->e_id, e->e_dn, 0 );
657 bdb = (struct bdb_info *) be->be_private;
659 if (! (slapMode & SLAP_TOOL_QUICK)) {
660 rc = TXN_BEGIN( bdb->bi_dbenv, NULL, &tid,
661 bdb->bi_db_opflags );
663 snprintf( text->bv_val, text->bv_len,
664 "txn_begin failed: %s (%d)",
665 db_strerror(rc), rc );
666 Debug( LDAP_DEBUG_ANY,
667 "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
668 text->bv_val, 0, 0 );
675 op.o_tmpmemctx = NULL;
676 op.o_tmpmfuncs = &ch_mfuncs;
678 /* add dn2id indices */
679 rc = bdb_tool_next_id( &op, tid, e, text, 0 );
685 if (( slapMode & SLAP_TOOL_QUICK ) && (( e->e_id & 0xfff ) == 0xfff )) {
686 ldap_pvt_thread_cond_signal( &bdb_tool_trickle_cond );
690 if ( !bdb->bi_linear_index )
691 rc = bdb_tool_index_add( &op, tid, e );
693 snprintf( text->bv_val, text->bv_len,
694 "index_entry_add failed: %s (%d)",
695 rc == LDAP_OTHER ? "Internal error" :
696 db_strerror(rc), rc );
697 Debug( LDAP_DEBUG_ANY,
698 "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
699 text->bv_val, 0, 0 );
704 rc = bdb_id2entry_add( be, tid, e );
706 snprintf( text->bv_val, text->bv_len,
707 "id2entry_add failed: %s (%d)",
708 db_strerror(rc), rc );
709 Debug( LDAP_DEBUG_ANY,
710 "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
711 text->bv_val, 0, 0 );
717 if ( !( slapMode & SLAP_TOOL_QUICK )) {
718 rc = TXN_COMMIT( tid, 0 );
720 snprintf( text->bv_val, text->bv_len,
721 "txn_commit failed: %s (%d)",
722 db_strerror(rc), rc );
723 Debug( LDAP_DEBUG_ANY,
724 "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
725 text->bv_val, 0, 0 );
731 if ( !( slapMode & SLAP_TOOL_QUICK )) {
733 snprintf( text->bv_val, text->bv_len,
734 "txn_aborted! %s (%d)",
735 rc == LDAP_OTHER ? "Internal error" :
736 db_strerror(rc), rc );
737 Debug( LDAP_DEBUG_ANY,
738 "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
739 text->bv_val, 0, 0 );
747 int bdb_tool_entry_reindex(
750 AttributeDescription **adv )
752 struct bdb_info *bi = (struct bdb_info *) be->be_private;
759 Debug( LDAP_DEBUG_ARGS,
760 "=> " LDAP_XSTRING(bdb_tool_entry_reindex) "( %ld )\n",
762 assert( tool_base == NULL );
763 assert( tool_filter == NULL );
765 /* No indexes configured, nothing to do. Could return an
766 * error here to shortcut things.
772 /* Check for explicit list of attrs to index */
776 if ( bi->bi_attrs[0]->ai_desc != adv[0] ) {
778 for ( n = 0; adv[n]; n++ ) ;
781 for ( i = 0; i < n; i++ ) {
782 AttributeDescription *ad = adv[i];
783 for ( j = i-1; j>=0; j--) {
784 if ( SLAP_PTRCMP( adv[j], ad ) <= 0 ) break;
791 for ( i = 0; adv[i]; i++ ) {
792 if ( bi->bi_attrs[i]->ai_desc != adv[i] ) {
793 for ( j = i+1; j < bi->bi_nattrs; j++ ) {
794 if ( bi->bi_attrs[j]->ai_desc == adv[i] ) {
795 AttrInfo *ai = bi->bi_attrs[i];
796 bi->bi_attrs[i] = bi->bi_attrs[j];
797 bi->bi_attrs[j] = ai;
801 if ( j == bi->bi_nattrs ) {
802 Debug( LDAP_DEBUG_ANY,
803 LDAP_XSTRING(bdb_tool_entry_reindex)
804 ": no index configured for %s\n",
805 adv[i]->ad_cname.bv_val, 0, 0 );
813 /* Get the first attribute to index */
814 if (bi->bi_linear_index && !index_nattrs) {
815 index_nattrs = bi->bi_nattrs - 1;
819 e = bdb_tool_entry_get( be, id );
822 Debug( LDAP_DEBUG_ANY,
823 LDAP_XSTRING(bdb_tool_entry_reindex)
824 ": could not locate id=%ld\n",
829 if (! (slapMode & SLAP_TOOL_QUICK)) {
830 rc = TXN_BEGIN( bi->bi_dbenv, NULL, &tid, bi->bi_db_opflags );
832 Debug( LDAP_DEBUG_ANY,
833 "=> " LDAP_XSTRING(bdb_tool_entry_reindex) ": "
834 "txn_begin failed: %s (%d)\n",
835 db_strerror(rc), rc, 0 );
841 * just (re)add them for now
842 * assume that some other routine (not yet implemented)
843 * will zap index databases
847 Debug( LDAP_DEBUG_TRACE,
848 "=> " LDAP_XSTRING(bdb_tool_entry_reindex) "( %ld, \"%s\" )\n",
849 (long) id, e->e_dn, 0 );
853 op.o_tmpmemctx = NULL;
854 op.o_tmpmfuncs = &ch_mfuncs;
856 rc = bdb_tool_index_add( &op, tid, e );
860 if (! (slapMode & SLAP_TOOL_QUICK)) {
861 rc = TXN_COMMIT( tid, 0 );
863 Debug( LDAP_DEBUG_ANY,
864 "=> " LDAP_XSTRING(bdb_tool_entry_reindex)
865 ": txn_commit failed: %s (%d)\n",
866 db_strerror(rc), rc, 0 );
872 if (! (slapMode & SLAP_TOOL_QUICK)) {
874 Debug( LDAP_DEBUG_ANY,
875 "=> " LDAP_XSTRING(bdb_tool_entry_reindex)
876 ": txn_aborted! %s (%d)\n",
877 db_strerror(rc), rc, 0 );
881 bdb_entry_release( &op, e, 0 );
886 ID bdb_tool_entry_modify(
889 struct berval *text )
892 struct bdb_info *bdb;
897 assert( be != NULL );
898 assert( slapMode & SLAP_TOOL_MODE );
900 assert( text != NULL );
901 assert( text->bv_val != NULL );
902 assert( text->bv_val[0] == '\0' ); /* overconservative? */
904 assert ( e->e_id != NOID );
906 Debug( LDAP_DEBUG_TRACE,
907 "=> " LDAP_XSTRING(bdb_tool_entry_modify) "( %ld, \"%s\" )\n",
908 (long) e->e_id, e->e_dn, 0 );
910 bdb = (struct bdb_info *) be->be_private;
912 if (! (slapMode & SLAP_TOOL_QUICK)) {
914 cursor->c_close( cursor );
917 rc = TXN_BEGIN( bdb->bi_dbenv, NULL, &tid,
918 bdb->bi_db_opflags );
920 snprintf( text->bv_val, text->bv_len,
921 "txn_begin failed: %s (%d)",
922 db_strerror(rc), rc );
923 Debug( LDAP_DEBUG_ANY,
924 "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n",
925 text->bv_val, 0, 0 );
932 op.o_tmpmemctx = NULL;
933 op.o_tmpmfuncs = &ch_mfuncs;
936 rc = bdb_id2entry_update( be, tid, e );
938 snprintf( text->bv_val, text->bv_len,
939 "id2entry_add failed: %s (%d)",
940 db_strerror(rc), rc );
941 Debug( LDAP_DEBUG_ANY,
942 "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n",
943 text->bv_val, 0, 0 );
949 if (! (slapMode & SLAP_TOOL_QUICK)) {
950 rc = TXN_COMMIT( tid, 0 );
952 snprintf( text->bv_val, text->bv_len,
953 "txn_commit failed: %s (%d)",
954 db_strerror(rc), rc );
955 Debug( LDAP_DEBUG_ANY,
956 "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": "
957 "%s\n", text->bv_val, 0, 0 );
963 if (! (slapMode & SLAP_TOOL_QUICK)) {
965 snprintf( text->bv_val, text->bv_len,
966 "txn_aborted! %s (%d)",
967 db_strerror(rc), rc );
968 Debug( LDAP_DEBUG_ANY,
969 "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n",
970 text->bv_val, 0, 0 );
978 #ifdef BDB_TOOL_IDL_CACHING
980 bdb_tool_idl_cmp( const void *v1, const void *v2 )
982 const bdb_tool_idl_cache *c1 = v1, *c2 = v2;
985 if (( rc = c1->kstr.bv_len - c2->kstr.bv_len )) return rc;
986 return memcmp( c1->kstr.bv_val, c2->kstr.bv_val, c1->kstr.bv_len );
990 bdb_tool_idl_flush_one( void *v1, void *arg )
992 bdb_tool_idl_cache *ic = v1;
994 struct bdb_info *bdb = bdb_tool_info;
995 bdb_tool_idl_cache_entry *ice;
1001 /* Freshly allocated, ignore it */
1002 if ( !ic->head && ic->count <= BDB_IDL_DB_SIZE ) {
1006 rc = db->cursor( db, NULL, &curs, 0 );
1013 bv2DBT( &ic->kstr, &key );
1015 data.size = data.ulen = sizeof( ID );
1016 data.flags = DB_DBT_USERMEM;
1019 rc = curs->c_get( curs, &key, &data, DB_SET );
1020 /* If key already exists and we're writing a range... */
1021 if ( rc == 0 && ic->count > BDB_IDL_DB_SIZE ) {
1022 /* If it's not currently a range, must delete old info */
1025 while ( curs->c_get( curs, &key, &data, DB_NEXT_DUP ) == 0 )
1026 curs->c_del( curs, 0 );
1029 /* Store range marker */
1030 curs->c_put( curs, &key, &data, DB_KEYFIRST );
1034 rc = curs->c_get( curs, &key, &data, DB_NEXT_DUP );
1037 rc = curs->c_get( curs, &key, &data, DB_NEXT_DUP );
1040 curs->c_del( curs, 0 );
1042 BDB_ID2DISK( ic->last, &nid );
1043 curs->c_put( curs, &key, &data, DB_KEYLAST );
1045 } else if ( rc && rc != DB_NOTFOUND ) {
1047 } else if ( ic->count > BDB_IDL_DB_SIZE ) {
1048 /* range, didn't exist before */
1050 rc = curs->c_put( curs, &key, &data, DB_KEYLAST );
1052 BDB_ID2DISK( ic->first, &nid );
1053 rc = curs->c_put( curs, &key, &data, DB_KEYLAST );
1055 BDB_ID2DISK( ic->last, &nid );
1056 rc = curs->c_put( curs, &key, &data, DB_KEYLAST );
1065 /* Just a normal write */
1067 for ( ice = ic->head, n=0; ice; ice = ice->next, n++ ) {
1072 end = ic->count & (IDBLOCK-1);
1076 for ( i=0; i<end; i++ ) {
1077 if ( !ice->ids[i] ) continue;
1078 BDB_ID2DISK( ice->ids[i], &nid );
1079 rc = curs->c_put( curs, &key, &data, DB_NODUPDATA );
1081 if ( rc == DB_KEYEXIST ) {
1095 ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
1096 ic->tail->next = bdb_tool_idl_free_list;
1097 bdb_tool_idl_free_list = ic->head;
1098 bdb->bi_idl_cache_size -= n;
1099 ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
1102 if ( ic != db->app_private ) {
1105 ic->head = ic->tail = NULL;
1107 curs->c_close( curs );
1112 bdb_tool_idl_flush_db( DB *db, bdb_tool_idl_cache *ic )
1114 Avlnode *root = db->app_private;
1117 db->app_private = ic;
1118 rc = avl_apply( root, bdb_tool_idl_flush_one, db, -1, AVL_INORDER );
1119 avl_free( root, NULL );
1120 db->app_private = NULL;
1127 bdb_tool_idl_flush( BackendDB *be )
1129 struct bdb_info *bdb = (struct bdb_info *) be->be_private;
1134 for ( i=BDB_NDB; i < bdb->bi_ndatabases; i++ ) {
1135 db = bdb->bi_databases[i]->bdi_db;
1136 if ( !db->app_private ) continue;
1137 rc = bdb_tool_idl_flush_db( db, NULL );
1142 bdb->bi_idl_cache_size = 0;
1147 int bdb_tool_idl_add(
1154 struct bdb_info *bdb = (struct bdb_info *) be->be_private;
1155 bdb_tool_idl_cache *ic, itmp;
1156 bdb_tool_idl_cache_entry *ice;
1159 if ( !bdb->bi_idl_cache_max_size )
1160 return bdb_idl_insert_key( be, db, txn, key, id );
1162 DBT2bv( key, &itmp.kstr );
1164 ic = avl_find( (Avlnode *)db->app_private, &itmp, bdb_tool_idl_cmp );
1166 /* No entry yet, create one */
1173 ic = ch_malloc( sizeof( bdb_tool_idl_cache ) + itmp.kstr.bv_len );
1174 ic->kstr.bv_len = itmp.kstr.bv_len;
1175 ic->kstr.bv_val = (char *)(ic+1);
1176 AC_MEMCPY( ic->kstr.bv_val, itmp.kstr.bv_val, ic->kstr.bv_len );
1177 ic->head = ic->tail = NULL;
1180 avl_insert( (Avlnode **)&db->app_private, ic, bdb_tool_idl_cmp,
1183 /* load existing key count here */
1184 rc = db->cursor( db, NULL, &curs, 0 );
1185 if ( rc ) return rc;
1187 data.ulen = sizeof( ID );
1188 data.flags = DB_DBT_USERMEM;
1190 rc = curs->c_get( curs, key, &data, DB_SET );
1193 ic->count = BDB_IDL_DB_SIZE+1;
1197 curs->c_count( curs, &count, 0 );
1199 BDB_DISK2ID( &nid, &ic->first );
1202 curs->c_close( curs );
1204 /* are we a range already? */
1205 if ( ic->count > BDB_IDL_DB_SIZE ) {
1208 /* Are we at the limit, and converting to a range? */
1209 } else if ( ic->count == BDB_IDL_DB_SIZE ) {
1211 for ( ice = ic->head, n=0; ice; ice = ice->next, n++ )
1214 ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
1215 ic->tail->next = bdb_tool_idl_free_list;
1216 bdb_tool_idl_free_list = ic->head;
1217 bdb->bi_idl_cache_size -= n;
1218 ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
1220 ic->head = ic->tail = NULL;
1225 /* No free block, create that too */
1226 if ( !ic->tail || ( ic->count & (IDBLOCK-1)) == 0) {
1228 ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
1229 if ( bdb->bi_idl_cache_size >= bdb->bi_idl_cache_max_size ) {
1230 ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
1231 rc = bdb_tool_idl_flush_db( db, ic );
1234 avl_insert( (Avlnode **)&db->app_private, ic, bdb_tool_idl_cmp,
1236 ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
1238 bdb->bi_idl_cache_size++;
1239 if ( bdb_tool_idl_free_list ) {
1240 ice = bdb_tool_idl_free_list;
1241 bdb_tool_idl_free_list = ice->next;
1243 ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
1245 ice = ch_malloc( sizeof( bdb_tool_idl_cache_entry ));
1247 memset( ice, 0, sizeof( *ice ));
1251 ic->tail->next = ice;
1258 ice->ids[ ic->count & (IDBLOCK-1) ] = id;
1267 bdb_tool_trickle_task( void *ctx, void *ptr )
1272 ldap_pvt_thread_mutex_lock( &bdb_tool_trickle_mutex );
1273 bdb_tool_trickle_active = 1;
1274 ldap_pvt_thread_cond_signal( &bdb_tool_trickle_cond_end );
1276 ldap_pvt_thread_cond_wait( &bdb_tool_trickle_cond,
1277 &bdb_tool_trickle_mutex );
1278 if ( slapd_shutdown )
1280 env->memp_trickle( env, 30, &wrote );
1282 bdb_tool_trickle_active = 0;
1283 ldap_pvt_thread_cond_signal( &bdb_tool_trickle_cond_end );
1284 ldap_pvt_thread_mutex_unlock( &bdb_tool_trickle_mutex );
1291 bdb_tool_index_task( void *ctx, void *ptr )
1293 int base = *(int *)ptr;
1297 ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex );
1298 bdb_tool_index_tcount--;
1299 if ( !bdb_tool_index_tcount )
1300 ldap_pvt_thread_cond_signal( &bdb_tool_index_cond_main );
1301 ldap_pvt_thread_cond_wait( &bdb_tool_index_cond_work,
1302 &bdb_tool_index_mutex );
1303 if ( slapd_shutdown ) {
1304 bdb_tool_index_tcount--;
1305 if ( !bdb_tool_index_tcount )
1306 ldap_pvt_thread_cond_signal( &bdb_tool_index_cond_main );
1307 ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex );
1310 ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex );
1312 bdb_tool_index_threads[base] = bdb_index_recrun( bdb_tool_ix_op,
1313 bdb_tool_info, bdb_tool_index_rec, bdb_tool_ix_id, base );