1 /* tools.c - tools for slap tools */
3 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
5 * Copyright 2000-2011 The OpenLDAP Foundation.
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted only as authorized by the OpenLDAP
12 * A copy of this license is available in the file LICENSE in the
13 * top-level directory of the distribution or, alternatively, at
14 * <http://www.OpenLDAP.org/license.html>.
20 #include <ac/string.h>
27 static DBC *cursor = NULL;
29 static EntryHeader eh;
30 static ID nid, previd = NOID;
31 static char ehbuf[16];
33 typedef struct dn_id {
38 #define HOLE_SIZE 4096
39 static dn_id hbuf[HOLE_SIZE], *holes = hbuf;
40 static unsigned nhmax = HOLE_SIZE;
41 static unsigned nholes;
43 static int index_nattrs;
45 static struct berval *tool_base;
46 static int tool_scope;
47 static Filter *tool_filter;
48 static Entry *tool_next_entry;
50 #ifdef BDB_TOOL_IDL_CACHING
51 #define bdb_tool_idl_cmp BDB_SYMBOL(tool_idl_cmp)
52 #define bdb_tool_idl_flush_one BDB_SYMBOL(tool_idl_flush_one)
53 #define bdb_tool_idl_flush BDB_SYMBOL(tool_idl_flush)
55 static int bdb_tool_idl_flush( BackendDB *be );
59 typedef struct bdb_tool_idl_cache_entry {
60 struct bdb_tool_idl_cache_entry *next;
62 } bdb_tool_idl_cache_entry;
64 typedef struct bdb_tool_idl_cache {
66 bdb_tool_idl_cache_entry *head, *tail;
71 static bdb_tool_idl_cache_entry *bdb_tool_idl_free_list;
72 #endif /* BDB_TOOL_IDL_CACHING */
74 static ID bdb_tool_ix_id;
75 static Operation *bdb_tool_ix_op;
76 static int *bdb_tool_index_threads, bdb_tool_index_tcount;
77 static void *bdb_tool_index_rec;
78 static struct bdb_info *bdb_tool_info;
79 static ldap_pvt_thread_mutex_t bdb_tool_index_mutex;
80 static ldap_pvt_thread_cond_t bdb_tool_index_cond_main;
81 static ldap_pvt_thread_cond_t bdb_tool_index_cond_work;
83 #if DB_VERSION_FULL >= 0x04060000
86 /* Seems to slow things down too much in BDB 4.5 */
91 static ldap_pvt_thread_mutex_t bdb_tool_trickle_mutex;
92 static ldap_pvt_thread_cond_t bdb_tool_trickle_cond;
94 static void * bdb_tool_trickle_task( void *ctx, void *ptr );
97 static void * bdb_tool_index_task( void *ctx, void *ptr );
100 bdb_tool_entry_get_int( BackendDB *be, ID id, Entry **ep );
102 int bdb_tool_entry_open(
103 BackendDB *be, int mode )
105 struct bdb_info *bdb = (struct bdb_info *) be->be_private;
107 /* initialize key and data thangs */
110 key.flags = DB_DBT_USERMEM;
112 key.size = key.ulen = sizeof( nid );
113 data.flags = DB_DBT_USERMEM;
115 if (cursor == NULL) {
116 int rc = bdb->bi_id2entry->bdi_db->cursor(
117 bdb->bi_id2entry->bdi_db, bdb->bi_cache.c_txn, &cursor,
118 bdb->bi_db_opflags );
124 /* Set up for threaded slapindex */
125 if (( slapMode & (SLAP_TOOL_QUICK|SLAP_TOOL_READONLY)) == SLAP_TOOL_QUICK ) {
126 if ( !bdb_tool_info ) {
128 ldap_pvt_thread_mutex_init( &bdb_tool_trickle_mutex );
129 ldap_pvt_thread_cond_init( &bdb_tool_trickle_cond );
130 ldap_pvt_thread_pool_submit( &connection_pool, bdb_tool_trickle_task, bdb->bi_dbenv );
133 ldap_pvt_thread_mutex_init( &bdb_tool_index_mutex );
134 ldap_pvt_thread_cond_init( &bdb_tool_index_cond_main );
135 ldap_pvt_thread_cond_init( &bdb_tool_index_cond_work );
136 if ( bdb->bi_nattrs ) {
138 bdb_tool_index_threads = ch_malloc( slap_tool_thread_max * sizeof( int ));
139 bdb_tool_index_rec = ch_malloc( bdb->bi_nattrs * sizeof( IndexRec ));
140 bdb_tool_index_tcount = slap_tool_thread_max - 1;
141 for (i=1; i<slap_tool_thread_max; i++) {
142 int *ptr = ch_malloc( sizeof( int ));
144 ldap_pvt_thread_pool_submit( &connection_pool,
145 bdb_tool_index_task, ptr );
155 int bdb_tool_entry_close(
158 if ( bdb_tool_info ) {
161 ldap_pvt_thread_mutex_lock( &bdb_tool_trickle_mutex );
162 ldap_pvt_thread_cond_signal( &bdb_tool_trickle_cond );
163 ldap_pvt_thread_mutex_unlock( &bdb_tool_trickle_mutex );
165 ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex );
167 /* There might still be some threads starting */
168 while ( bdb_tool_index_tcount ) {
169 ldap_pvt_thread_cond_wait( &bdb_tool_index_cond_main,
170 &bdb_tool_index_mutex );
173 bdb_tool_index_tcount = slap_tool_thread_max - 1;
174 ldap_pvt_thread_cond_broadcast( &bdb_tool_index_cond_work );
176 /* Make sure all threads are stopped */
177 while ( bdb_tool_index_tcount ) {
178 ldap_pvt_thread_cond_wait( &bdb_tool_index_cond_main,
179 &bdb_tool_index_mutex );
181 ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex );
183 bdb_tool_info = NULL;
185 ch_free( bdb_tool_index_threads );
186 ch_free( bdb_tool_index_rec );
187 bdb_tool_index_tcount = slap_tool_thread_max - 1;
191 ch_free( eh.bv.bv_val );
196 cursor->c_close( cursor );
200 #ifdef BDB_TOOL_IDL_CACHING
201 bdb_tool_idl_flush( be );
206 fprintf( stderr, "Error, entries missing!\n");
207 for (i=0; i<nholes; i++) {
208 fprintf(stderr, " entry %ld: %s\n",
209 holes[i].id, holes[i].dn.bv_val);
218 bdb_tool_entry_first_x(
228 return bdb_tool_entry_next( be );
231 ID bdb_tool_entry_next(
236 struct bdb_info *bdb = (struct bdb_info *) be->be_private;
238 assert( be != NULL );
239 assert( slapMode & SLAP_TOOL_MODE );
240 assert( bdb != NULL );
244 data.ulen = data.dlen = sizeof( ehbuf );
246 data.flags |= DB_DBT_PARTIAL;
247 rc = cursor->c_get( cursor, &key, &data, DB_NEXT );
250 /* If we're doing linear indexing and there are more attrs to
251 * index, and we're at the end of the database, start over.
253 if ( index_nattrs && rc == DB_NOTFOUND ) {
254 /* optional - do a checkpoint here? */
255 bdb_attr_info_free( bdb->bi_attrs[0] );
256 bdb->bi_attrs[0] = bdb->bi_attrs[index_nattrs];
258 rc = cursor->c_get( cursor, &key, &data, DB_FIRST );
267 BDB_DISK2ID( key.data, &id );
270 if ( tool_filter || tool_base ) {
271 static Operation op = {0};
272 static Opheader ohdr = {0};
276 op.o_tmpmemctx = NULL;
277 op.o_tmpmfuncs = &ch_mfuncs;
279 if ( tool_next_entry ) {
280 bdb_entry_release( &op, tool_next_entry, 0 );
281 tool_next_entry = NULL;
284 rc = bdb_tool_entry_get_int( be, id, &tool_next_entry );
285 if ( rc == LDAP_NO_SUCH_OBJECT ) {
289 assert( tool_next_entry != NULL );
292 /* TODO: needed until BDB_HIER is handled accordingly
293 * in bdb_tool_entry_get_int() */
294 if ( tool_base && !dnIsSuffixScope( &tool_next_entry->e_nname, tool_base, tool_scope ) )
296 bdb_entry_release( &op, tool_next_entry, 0 );
297 tool_next_entry = NULL;
302 if ( tool_filter && test_filter( NULL, tool_next_entry, tool_filter ) != LDAP_COMPARE_TRUE )
304 bdb_entry_release( &op, tool_next_entry, 0 );
305 tool_next_entry = NULL;
313 ID bdb_tool_dn2id_get(
320 EntryInfo *ei = NULL;
323 if ( BER_BVISEMPTY(dn) )
328 op.o_tmpmemctx = NULL;
329 op.o_tmpmfuncs = &ch_mfuncs;
331 rc = bdb_cache_find_ndn( &op, 0, dn, &ei );
332 if ( ei ) bdb_cache_entryinfo_unlock( ei );
333 if ( rc == DB_NOTFOUND )
340 bdb_tool_entry_get_int( BackendDB *be, ID id, Entry **ep )
346 assert( be != NULL );
347 assert( slapMode & SLAP_TOOL_MODE );
349 if ( ( tool_filter || tool_base ) && id == previd && tool_next_entry != NULL ) {
350 *ep = tool_next_entry;
351 tool_next_entry = NULL;
355 if ( id != previd ) {
356 data.ulen = data.dlen = sizeof( ehbuf );
358 data.flags |= DB_DBT_PARTIAL;
360 BDB_ID2DISK( id, &nid );
361 rc = cursor->c_get( cursor, &key, &data, DB_SET );
370 eh.bv.bv_val = ehbuf;
371 eh.bv.bv_len = data.size;
372 rc = entry_header( &eh );
373 eoff = eh.data - eh.bv.bv_val;
381 data.flags &= ~DB_DBT_PARTIAL;
383 rc = cursor->c_get( cursor, &key, &data, DB_CURRENT );
384 if ( rc != DB_BUFFER_SMALL ) {
389 /* Allocate a block and retrieve the data */
390 eh.bv.bv_len = eh.nvals * sizeof( struct berval ) + data.size;
391 eh.bv.bv_val = ch_realloc( eh.bv.bv_val, eh.bv.bv_len );
392 eh.data = eh.bv.bv_val + eh.nvals * sizeof( struct berval );
394 data.ulen = data.size;
396 /* Skip past already parsed nattr/nvals */
399 rc = cursor->c_get( cursor, &key, &data, DB_CURRENT );
406 /* TODO: handle BDB_HIER accordingly */
407 if ( tool_base != NULL ) {
409 entry_decode_dn( &eh, NULL, &ndn );
411 if ( !dnIsSuffixScope( &ndn, tool_base, tool_scope ) ) {
412 return LDAP_NO_SUCH_OBJECT;
417 #ifdef SLAP_ZONE_ALLOC
418 /* FIXME: will add ctx later */
419 rc = entry_decode( &eh, &e, NULL );
421 rc = entry_decode( &eh, &e );
424 if( rc == LDAP_SUCCESS ) {
427 if ( slapMode & SLAP_TOOL_READONLY ) {
428 struct bdb_info *bdb = (struct bdb_info *) be->be_private;
429 EntryInfo *ei = NULL;
435 op.o_tmpmemctx = NULL;
436 op.o_tmpmfuncs = &ch_mfuncs;
438 rc = bdb_cache_find_parent( &op, bdb->bi_cache.c_txn, id, &ei );
439 if ( rc == LDAP_SUCCESS ) {
440 bdb_cache_entryinfo_unlock( ei );
459 bdb_tool_entry_get( BackendDB *be, ID id )
463 (void)bdb_tool_entry_get_int( be, id, &e );
467 static int bdb_tool_next_id(
474 struct berval dn = e->e_name;
475 struct berval ndn = e->e_nname;
476 struct berval pdn, npdn;
477 EntryInfo *ei = NULL, eidummy;
480 if (ndn.bv_len == 0) {
485 rc = bdb_cache_find_ndn( op, tid, &ndn, &ei );
486 if ( ei ) bdb_cache_entryinfo_unlock( ei );
487 if ( rc == DB_NOTFOUND ) {
488 if ( !be_issuffix( op->o_bd, &ndn ) ) {
490 dnParent( &dn, &pdn );
491 dnParent( &ndn, &npdn );
494 rc = bdb_tool_next_id( op, tid, e, text, 1 );
500 /* If parent didn't exist, it was created just now
501 * and its ID is now in e->e_id. Make sure the current
502 * entry gets added under the new parent ID.
504 if ( eid != e->e_id ) {
505 eidummy.bei_id = e->e_id;
509 rc = bdb_next_id( op->o_bd, &e->e_id );
511 snprintf( text->bv_val, text->bv_len,
512 "next_id failed: %s (%d)",
513 db_strerror(rc), rc );
514 Debug( LDAP_DEBUG_ANY,
515 "=> bdb_tool_next_id: %s\n", text->bv_val, 0, 0 );
518 rc = bdb_dn2id_add( op, tid, ei, e );
520 snprintf( text->bv_val, text->bv_len,
521 "dn2id_add failed: %s (%d)",
522 db_strerror(rc), rc );
523 Debug( LDAP_DEBUG_ANY,
524 "=> bdb_tool_next_id: %s\n", text->bv_val, 0, 0 );
526 if ( nholes == nhmax - 1 ) {
527 if ( holes == hbuf ) {
528 holes = ch_malloc( nhmax * sizeof(dn_id) * 2 );
529 AC_MEMCPY( holes, hbuf, sizeof(hbuf) );
531 holes = ch_realloc( holes, nhmax * sizeof(dn_id) * 2 );
535 ber_dupbv( &holes[nholes].dn, &ndn );
536 holes[nholes++].id = e->e_id;
538 } else if ( !hole ) {
541 e->e_id = ei->bei_id;
543 for ( i=0; i<nholes; i++) {
544 if ( holes[i].id == e->e_id ) {
545 free(holes[i].dn.bv_val);
546 for (j=i;j<nholes;j++) holes[j] = holes[j+1];
550 } else if ( holes[i].id > e->e_id ) {
564 struct bdb_info *bdb = (struct bdb_info *) op->o_bd->be_private;
566 if ( !bdb->bi_nattrs )
569 if ( slapMode & SLAP_TOOL_QUICK ) {
574 ir = bdb_tool_index_rec;
575 memset(ir, 0, bdb->bi_nattrs * sizeof( IndexRec ));
577 for ( a = e->e_attrs; a != NULL; a = a->a_next ) {
578 rc = bdb_index_recset( bdb, a, a->a_desc->ad_type,
579 &a->a_desc->ad_tags, ir );
583 bdb_tool_ix_id = e->e_id;
585 ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex );
586 /* Wait for all threads to be ready */
587 while ( bdb_tool_index_tcount ) {
588 ldap_pvt_thread_cond_wait( &bdb_tool_index_cond_main,
589 &bdb_tool_index_mutex );
591 for ( i=1; i<slap_tool_thread_max; i++ )
592 bdb_tool_index_threads[i] = LDAP_BUSY;
593 bdb_tool_index_tcount = slap_tool_thread_max - 1;
594 ldap_pvt_thread_cond_broadcast( &bdb_tool_index_cond_work );
595 ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex );
596 rc = bdb_index_recrun( op, bdb, ir, e->e_id, 0 );
599 ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex );
600 for ( i=1; i<slap_tool_thread_max; i++ ) {
601 if ( bdb_tool_index_threads[i] == LDAP_BUSY ) {
602 ldap_pvt_thread_cond_wait( &bdb_tool_index_cond_main,
603 &bdb_tool_index_mutex );
607 if ( bdb_tool_index_threads[i] ) {
608 rc = bdb_tool_index_threads[i];
612 ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex );
615 return bdb_index_entry_add( op, txn, e );
619 ID bdb_tool_entry_put(
622 struct berval *text )
625 struct bdb_info *bdb = (struct bdb_info *) be->be_private;
630 assert( be != NULL );
631 assert( slapMode & SLAP_TOOL_MODE );
633 assert( text != NULL );
634 assert( text->bv_val != NULL );
635 assert( text->bv_val[0] == '\0' ); /* overconservative? */
637 Debug( LDAP_DEBUG_TRACE, "=> " LDAP_XSTRING(bdb_tool_entry_put)
638 "( %ld, \"%s\" )\n", (long) e->e_id, e->e_dn, 0 );
640 if (! (slapMode & SLAP_TOOL_QUICK)) {
641 rc = TXN_BEGIN( bdb->bi_dbenv, NULL, &tid,
642 bdb->bi_db_opflags );
644 snprintf( text->bv_val, text->bv_len,
645 "txn_begin failed: %s (%d)",
646 db_strerror(rc), rc );
647 Debug( LDAP_DEBUG_ANY,
648 "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
649 text->bv_val, 0, 0 );
656 op.o_tmpmemctx = NULL;
657 op.o_tmpmfuncs = &ch_mfuncs;
659 /* add dn2id indices */
660 rc = bdb_tool_next_id( &op, tid, e, text, 0 );
666 if (( slapMode & SLAP_TOOL_QUICK ) && (( e->e_id & 0xfff ) == 0xfff )) {
667 ldap_pvt_thread_cond_signal( &bdb_tool_trickle_cond );
671 if ( !bdb->bi_linear_index )
672 rc = bdb_tool_index_add( &op, tid, e );
674 snprintf( text->bv_val, text->bv_len,
675 "index_entry_add failed: %s (%d)",
676 rc == LDAP_OTHER ? "Internal error" :
677 db_strerror(rc), rc );
678 Debug( LDAP_DEBUG_ANY,
679 "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
680 text->bv_val, 0, 0 );
685 rc = bdb_id2entry_add( be, tid, e );
687 snprintf( text->bv_val, text->bv_len,
688 "id2entry_add failed: %s (%d)",
689 db_strerror(rc), rc );
690 Debug( LDAP_DEBUG_ANY,
691 "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
692 text->bv_val, 0, 0 );
698 if ( !( slapMode & SLAP_TOOL_QUICK )) {
699 rc = TXN_COMMIT( tid, 0 );
701 snprintf( text->bv_val, text->bv_len,
702 "txn_commit failed: %s (%d)",
703 db_strerror(rc), rc );
704 Debug( LDAP_DEBUG_ANY,
705 "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
706 text->bv_val, 0, 0 );
712 if ( !( slapMode & SLAP_TOOL_QUICK )) {
714 snprintf( text->bv_val, text->bv_len,
715 "txn_aborted! %s (%d)",
716 rc == LDAP_OTHER ? "Internal error" :
717 db_strerror(rc), rc );
718 Debug( LDAP_DEBUG_ANY,
719 "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
720 text->bv_val, 0, 0 );
728 int bdb_tool_entry_reindex(
731 AttributeDescription **adv )
733 struct bdb_info *bi = (struct bdb_info *) be->be_private;
740 Debug( LDAP_DEBUG_ARGS,
741 "=> " LDAP_XSTRING(bdb_tool_entry_reindex) "( %ld )\n",
743 assert( tool_base == NULL );
744 assert( tool_filter == NULL );
746 /* No indexes configured, nothing to do. Could return an
747 * error here to shortcut things.
753 /* Check for explicit list of attrs to index */
757 if ( bi->bi_attrs[0]->ai_desc != adv[0] ) {
759 for ( n = 0; adv[n]; n++ ) ;
762 for ( i = 0; i < n; i++ ) {
763 AttributeDescription *ad = adv[i];
764 for ( j = i-1; j>=0; j--) {
765 if ( SLAP_PTRCMP( adv[j], ad ) <= 0 ) break;
772 for ( i = 0; adv[i]; i++ ) {
773 if ( bi->bi_attrs[i]->ai_desc != adv[i] ) {
774 for ( j = i+1; j < bi->bi_nattrs; j++ ) {
775 if ( bi->bi_attrs[j]->ai_desc == adv[i] ) {
776 AttrInfo *ai = bi->bi_attrs[i];
777 bi->bi_attrs[i] = bi->bi_attrs[j];
778 bi->bi_attrs[j] = ai;
782 if ( j == bi->bi_nattrs ) {
783 Debug( LDAP_DEBUG_ANY,
784 LDAP_XSTRING(bdb_tool_entry_reindex)
785 ": no index configured for %s\n",
786 adv[i]->ad_cname.bv_val, 0, 0 );
794 /* Get the first attribute to index */
795 if (bi->bi_linear_index && !index_nattrs) {
796 index_nattrs = bi->bi_nattrs - 1;
800 e = bdb_tool_entry_get( be, id );
803 Debug( LDAP_DEBUG_ANY,
804 LDAP_XSTRING(bdb_tool_entry_reindex)
805 ": could not locate id=%ld\n",
810 if (! (slapMode & SLAP_TOOL_QUICK)) {
811 rc = TXN_BEGIN( bi->bi_dbenv, NULL, &tid, bi->bi_db_opflags );
813 Debug( LDAP_DEBUG_ANY,
814 "=> " LDAP_XSTRING(bdb_tool_entry_reindex) ": "
815 "txn_begin failed: %s (%d)\n",
816 db_strerror(rc), rc, 0 );
822 * just (re)add them for now
823 * assume that some other routine (not yet implemented)
824 * will zap index databases
828 Debug( LDAP_DEBUG_TRACE,
829 "=> " LDAP_XSTRING(bdb_tool_entry_reindex) "( %ld, \"%s\" )\n",
830 (long) id, e->e_dn, 0 );
834 op.o_tmpmemctx = NULL;
835 op.o_tmpmfuncs = &ch_mfuncs;
837 rc = bdb_tool_index_add( &op, tid, e );
841 if (! (slapMode & SLAP_TOOL_QUICK)) {
842 rc = TXN_COMMIT( tid, 0 );
844 Debug( LDAP_DEBUG_ANY,
845 "=> " LDAP_XSTRING(bdb_tool_entry_reindex)
846 ": txn_commit failed: %s (%d)\n",
847 db_strerror(rc), rc, 0 );
853 if (! (slapMode & SLAP_TOOL_QUICK)) {
855 Debug( LDAP_DEBUG_ANY,
856 "=> " LDAP_XSTRING(bdb_tool_entry_reindex)
857 ": txn_aborted! %s (%d)\n",
858 db_strerror(rc), rc, 0 );
862 bdb_entry_release( &op, e, 0 );
867 ID bdb_tool_entry_modify(
870 struct berval *text )
873 struct bdb_info *bdb = (struct bdb_info *) be->be_private;
878 assert( be != NULL );
879 assert( slapMode & SLAP_TOOL_MODE );
881 assert( text != NULL );
882 assert( text->bv_val != NULL );
883 assert( text->bv_val[0] == '\0' ); /* overconservative? */
885 assert ( e->e_id != NOID );
887 Debug( LDAP_DEBUG_TRACE,
888 "=> " LDAP_XSTRING(bdb_tool_entry_modify) "( %ld, \"%s\" )\n",
889 (long) e->e_id, e->e_dn, 0 );
891 if (! (slapMode & SLAP_TOOL_QUICK)) {
893 cursor->c_close( cursor );
896 rc = TXN_BEGIN( bdb->bi_dbenv, NULL, &tid,
897 bdb->bi_db_opflags );
899 snprintf( text->bv_val, text->bv_len,
900 "txn_begin failed: %s (%d)",
901 db_strerror(rc), rc );
902 Debug( LDAP_DEBUG_ANY,
903 "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n",
904 text->bv_val, 0, 0 );
911 op.o_tmpmemctx = NULL;
912 op.o_tmpmfuncs = &ch_mfuncs;
915 rc = bdb_id2entry_update( be, tid, e );
917 snprintf( text->bv_val, text->bv_len,
918 "id2entry_add failed: %s (%d)",
919 db_strerror(rc), rc );
920 Debug( LDAP_DEBUG_ANY,
921 "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n",
922 text->bv_val, 0, 0 );
928 if (! (slapMode & SLAP_TOOL_QUICK)) {
929 rc = TXN_COMMIT( tid, 0 );
931 snprintf( text->bv_val, text->bv_len,
932 "txn_commit failed: %s (%d)",
933 db_strerror(rc), rc );
934 Debug( LDAP_DEBUG_ANY,
935 "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": "
936 "%s\n", text->bv_val, 0, 0 );
942 if (! (slapMode & SLAP_TOOL_QUICK)) {
944 snprintf( text->bv_val, text->bv_len,
945 "txn_aborted! %s (%d)",
946 db_strerror(rc), rc );
947 Debug( LDAP_DEBUG_ANY,
948 "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n",
949 text->bv_val, 0, 0 );
957 #ifdef BDB_TOOL_IDL_CACHING
959 bdb_tool_idl_cmp( const void *v1, const void *v2 )
961 const bdb_tool_idl_cache *c1 = v1, *c2 = v2;
964 if (( rc = c1->kstr.bv_len - c2->kstr.bv_len )) return rc;
965 return memcmp( c1->kstr.bv_val, c2->kstr.bv_val, c1->kstr.bv_len );
969 bdb_tool_idl_flush_one( void *v1, void *arg )
971 bdb_tool_idl_cache *ic = v1;
973 struct bdb_info *bdb = bdb_tool_info;
974 bdb_tool_idl_cache_entry *ice;
980 /* Freshly allocated, ignore it */
981 if ( !ic->head && ic->count <= BDB_IDL_DB_SIZE ) {
985 rc = db->cursor( db, NULL, &curs, 0 );
992 bv2DBT( &ic->kstr, &key );
994 data.size = data.ulen = sizeof( ID );
995 data.flags = DB_DBT_USERMEM;
998 rc = curs->c_get( curs, &key, &data, DB_SET );
999 /* If key already exists and we're writing a range... */
1000 if ( rc == 0 && ic->count > BDB_IDL_DB_SIZE ) {
1001 /* If it's not currently a range, must delete old info */
1004 while ( curs->c_get( curs, &key, &data, DB_NEXT_DUP ) == 0 )
1005 curs->c_del( curs, 0 );
1008 /* Store range marker */
1009 curs->c_put( curs, &key, &data, DB_KEYFIRST );
1013 rc = curs->c_get( curs, &key, &data, DB_NEXT_DUP );
1016 rc = curs->c_get( curs, &key, &data, DB_NEXT_DUP );
1019 curs->c_del( curs, 0 );
1021 BDB_ID2DISK( ic->last, &nid );
1022 curs->c_put( curs, &key, &data, DB_KEYLAST );
1024 } else if ( rc && rc != DB_NOTFOUND ) {
1026 } else if ( ic->count > BDB_IDL_DB_SIZE ) {
1027 /* range, didn't exist before */
1029 rc = curs->c_put( curs, &key, &data, DB_KEYLAST );
1031 BDB_ID2DISK( ic->first, &nid );
1032 rc = curs->c_put( curs, &key, &data, DB_KEYLAST );
1034 BDB_ID2DISK( ic->last, &nid );
1035 rc = curs->c_put( curs, &key, &data, DB_KEYLAST );
1044 /* Just a normal write */
1046 for ( ice = ic->head, n=0; ice; ice = ice->next, n++ ) {
1051 end = ic->count & (IDBLOCK-1);
1055 for ( i=0; i<end; i++ ) {
1056 if ( !ice->ids[i] ) continue;
1057 BDB_ID2DISK( ice->ids[i], &nid );
1058 rc = curs->c_put( curs, &key, &data, DB_NODUPDATA );
1060 if ( rc == DB_KEYEXIST ) {
1074 ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
1075 ic->tail->next = bdb_tool_idl_free_list;
1076 bdb_tool_idl_free_list = ic->head;
1077 bdb->bi_idl_cache_size -= n;
1078 ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
1081 if ( ic != db->app_private ) {
1084 ic->head = ic->tail = NULL;
1086 curs->c_close( curs );
1091 bdb_tool_idl_flush_db( DB *db, bdb_tool_idl_cache *ic )
1093 Avlnode *root = db->app_private;
1096 db->app_private = ic;
1097 rc = avl_apply( root, bdb_tool_idl_flush_one, db, -1, AVL_INORDER );
1098 avl_free( root, NULL );
1099 db->app_private = NULL;
1106 bdb_tool_idl_flush( BackendDB *be )
1108 struct bdb_info *bdb = (struct bdb_info *) be->be_private;
1113 for ( i=BDB_NDB; i < bdb->bi_ndatabases; i++ ) {
1114 db = bdb->bi_databases[i]->bdi_db;
1115 if ( !db->app_private ) continue;
1116 rc = bdb_tool_idl_flush_db( db, NULL );
1121 bdb->bi_idl_cache_size = 0;
1126 int bdb_tool_idl_add(
1133 struct bdb_info *bdb = (struct bdb_info *) be->be_private;
1134 bdb_tool_idl_cache *ic, itmp;
1135 bdb_tool_idl_cache_entry *ice;
1138 if ( !bdb->bi_idl_cache_max_size )
1139 return bdb_idl_insert_key( be, db, txn, key, id );
1141 DBT2bv( key, &itmp.kstr );
1143 ic = avl_find( (Avlnode *)db->app_private, &itmp, bdb_tool_idl_cmp );
1145 /* No entry yet, create one */
1152 ic = ch_malloc( sizeof( bdb_tool_idl_cache ) + itmp.kstr.bv_len );
1153 ic->kstr.bv_len = itmp.kstr.bv_len;
1154 ic->kstr.bv_val = (char *)(ic+1);
1155 AC_MEMCPY( ic->kstr.bv_val, itmp.kstr.bv_val, ic->kstr.bv_len );
1156 ic->head = ic->tail = NULL;
1159 avl_insert( (Avlnode **)&db->app_private, ic, bdb_tool_idl_cmp,
1162 /* load existing key count here */
1163 rc = db->cursor( db, NULL, &curs, 0 );
1164 if ( rc ) return rc;
1166 data.ulen = sizeof( ID );
1167 data.flags = DB_DBT_USERMEM;
1169 rc = curs->c_get( curs, key, &data, DB_SET );
1172 ic->count = BDB_IDL_DB_SIZE+1;
1176 curs->c_count( curs, &count, 0 );
1178 BDB_DISK2ID( &nid, &ic->first );
1181 curs->c_close( curs );
1183 /* are we a range already? */
1184 if ( ic->count > BDB_IDL_DB_SIZE ) {
1187 /* Are we at the limit, and converting to a range? */
1188 } else if ( ic->count == BDB_IDL_DB_SIZE ) {
1190 for ( ice = ic->head, n=0; ice; ice = ice->next, n++ )
1193 ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
1194 ic->tail->next = bdb_tool_idl_free_list;
1195 bdb_tool_idl_free_list = ic->head;
1196 bdb->bi_idl_cache_size -= n;
1197 ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
1199 ic->head = ic->tail = NULL;
1204 /* No free block, create that too */
1205 if ( !ic->tail || ( ic->count & (IDBLOCK-1)) == 0) {
1207 ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
1208 if ( bdb->bi_idl_cache_size >= bdb->bi_idl_cache_max_size ) {
1209 ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
1210 rc = bdb_tool_idl_flush_db( db, ic );
1213 avl_insert( (Avlnode **)&db->app_private, ic, bdb_tool_idl_cmp,
1215 ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
1217 bdb->bi_idl_cache_size++;
1218 if ( bdb_tool_idl_free_list ) {
1219 ice = bdb_tool_idl_free_list;
1220 bdb_tool_idl_free_list = ice->next;
1222 ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
1224 ice = ch_malloc( sizeof( bdb_tool_idl_cache_entry ));
1226 memset( ice, 0, sizeof( *ice ));
1230 ic->tail->next = ice;
1237 ice->ids[ ic->count & (IDBLOCK-1) ] = id;
1246 bdb_tool_trickle_task( void *ctx, void *ptr )
1251 ldap_pvt_thread_mutex_lock( &bdb_tool_trickle_mutex );
1253 ldap_pvt_thread_cond_wait( &bdb_tool_trickle_cond,
1254 &bdb_tool_trickle_mutex );
1255 if ( slapd_shutdown )
1257 env->memp_trickle( env, 30, &wrote );
1259 ldap_pvt_thread_mutex_unlock( &bdb_tool_trickle_mutex );
1266 bdb_tool_index_task( void *ctx, void *ptr )
1268 int base = *(int *)ptr;
1272 ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex );
1273 bdb_tool_index_tcount--;
1274 if ( !bdb_tool_index_tcount )
1275 ldap_pvt_thread_cond_signal( &bdb_tool_index_cond_main );
1276 ldap_pvt_thread_cond_wait( &bdb_tool_index_cond_work,
1277 &bdb_tool_index_mutex );
1278 if ( slapd_shutdown ) {
1279 bdb_tool_index_tcount--;
1280 if ( !bdb_tool_index_tcount )
1281 ldap_pvt_thread_cond_signal( &bdb_tool_index_cond_main );
1282 ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex );
1285 ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex );
1287 bdb_tool_index_threads[base] = bdb_index_recrun( bdb_tool_ix_op,
1288 bdb_tool_info, bdb_tool_index_rec, bdb_tool_ix_id, base );