]> git.sur5r.net Git - openldap/blob - servers/slapd/back-bdb/tools.c
Plug mutex/rwlock leaks (destroy them)
[openldap] / servers / slapd / back-bdb / tools.c
1 /* tools.c - tools for slap tools */
2 /* $OpenLDAP$ */
3 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
4  *
5  * Copyright 2000-2010 The OpenLDAP Foundation.
6  * All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted only as authorized by the OpenLDAP
10  * Public License.
11  *
12  * A copy of this license is available in the file LICENSE in the
13  * top-level directory of the distribution or, alternatively, at
14  * <http://www.OpenLDAP.org/license.html>.
15  */
16
17 #include "portable.h"
18
19 #include <stdio.h>
20 #include <ac/string.h>
21 #include <ac/errno.h>
22
23 #define AVL_INTERNAL
24 #include "back-bdb.h"
25 #include "idl.h"
26
27 static DBC *cursor = NULL;
28 static DBT key, data;
29 static EntryHeader eh;
30 static ID nid, previd = NOID;
31 static char ehbuf[16];
32
33 typedef struct dn_id {
34         ID id;
35         struct berval dn;
36 } dn_id;
37
38 #define HOLE_SIZE       4096
39 static dn_id hbuf[HOLE_SIZE], *holes = hbuf;
40 static unsigned nhmax = HOLE_SIZE;
41 static unsigned nholes;
42
43 static int index_nattrs;
44
45 static struct berval    *tool_base;
46 static int              tool_scope;
47 static Filter           *tool_filter;
48 static Entry            *tool_next_entry;
49
50 #ifdef BDB_TOOL_IDL_CACHING
51 #define bdb_tool_idl_cmp                BDB_SYMBOL(tool_idl_cmp)
52 #define bdb_tool_idl_flush_one          BDB_SYMBOL(tool_idl_flush_one)
53 #define bdb_tool_idl_flush              BDB_SYMBOL(tool_idl_flush)
54
55 static int bdb_tool_idl_flush( BackendDB *be );
56
57 #define IDBLOCK 1024
58
59 typedef struct bdb_tool_idl_cache_entry {
60         struct bdb_tool_idl_cache_entry *next;
61         ID ids[IDBLOCK];
62 } bdb_tool_idl_cache_entry;
63  
64 typedef struct bdb_tool_idl_cache {
65         struct berval kstr;
66         bdb_tool_idl_cache_entry *head, *tail;
67         ID first, last;
68         int count;
69 } bdb_tool_idl_cache;
70
71 static bdb_tool_idl_cache_entry *bdb_tool_idl_free_list;
72 #endif  /* BDB_TOOL_IDL_CACHING */
73
74 static ID bdb_tool_ix_id;
75 static Operation *bdb_tool_ix_op;
76 static int *bdb_tool_index_threads, bdb_tool_index_tcount;
77 static void *bdb_tool_index_rec;
78 static struct bdb_info *bdb_tool_info;
79 static ldap_pvt_thread_mutex_t bdb_tool_index_mutex;
80 static ldap_pvt_thread_cond_t bdb_tool_index_cond_main;
81 static ldap_pvt_thread_cond_t bdb_tool_index_cond_work;
82
83 #if DB_VERSION_FULL >= 0x04060000
84 #define USE_TRICKLE     1
85 #else
86 /* Seems to slow things down too much in BDB 4.5 */
87 #undef USE_TRICKLE
88 #endif
89
90 #ifdef USE_TRICKLE
91 static ldap_pvt_thread_mutex_t bdb_tool_trickle_mutex;
92 static ldap_pvt_thread_cond_t bdb_tool_trickle_cond;
93
94 static void * bdb_tool_trickle_task( void *ctx, void *ptr );
95 #endif
96
97 static void * bdb_tool_index_task( void *ctx, void *ptr );
98
99 static int
100 bdb_tool_entry_get_int( BackendDB *be, ID id, Entry **ep );
101
102 int bdb_tool_entry_open(
103         BackendDB *be, int mode )
104 {
105         struct bdb_info *bdb = (struct bdb_info *) be->be_private;
106
107         /* initialize key and data thangs */
108         DBTzero( &key );
109         DBTzero( &data );
110         key.flags = DB_DBT_USERMEM;
111         key.data = &nid;
112         key.size = key.ulen = sizeof( nid );
113         data.flags = DB_DBT_USERMEM;
114
115         if (cursor == NULL) {
116                 int rc = bdb->bi_id2entry->bdi_db->cursor(
117                         bdb->bi_id2entry->bdi_db, bdb->bi_cache.c_txn, &cursor,
118                         bdb->bi_db_opflags );
119                 if( rc != 0 ) {
120                         return -1;
121                 }
122         }
123
124         /* Set up for threaded slapindex */
125         if (( slapMode & (SLAP_TOOL_QUICK|SLAP_TOOL_READONLY)) == SLAP_TOOL_QUICK ) {
126                 if ( !bdb_tool_info ) {
127 #ifdef USE_TRICKLE
128                         ldap_pvt_thread_mutex_init( &bdb_tool_trickle_mutex );
129                         ldap_pvt_thread_cond_init( &bdb_tool_trickle_cond );
130                         ldap_pvt_thread_pool_submit( &connection_pool, bdb_tool_trickle_task, bdb->bi_dbenv );
131 #endif
132
133                         ldap_pvt_thread_mutex_init( &bdb_tool_index_mutex );
134                         ldap_pvt_thread_cond_init( &bdb_tool_index_cond_main );
135                         ldap_pvt_thread_cond_init( &bdb_tool_index_cond_work );
136                         if ( bdb->bi_nattrs ) {
137                                 int i;
138                                 bdb_tool_index_threads = ch_malloc( slap_tool_thread_max * sizeof( int ));
139                                 bdb_tool_index_rec = ch_malloc( bdb->bi_nattrs * sizeof( IndexRec ));
140                                 bdb_tool_index_tcount = slap_tool_thread_max - 1;
141                                 for (i=1; i<slap_tool_thread_max; i++) {
142                                         int *ptr = ch_malloc( sizeof( int ));
143                                         *ptr = i;
144                                         ldap_pvt_thread_pool_submit( &connection_pool,
145                                                 bdb_tool_index_task, ptr );
146                                 }
147                         }
148                         bdb_tool_info = bdb;
149                 }
150         }
151
152         return 0;
153 }
154
155 int bdb_tool_entry_close(
156         BackendDB *be )
157 {
158         if ( bdb_tool_info ) {
159                 slapd_shutdown = 1;
160 #ifdef USE_TRICKLE
161                 ldap_pvt_thread_mutex_lock( &bdb_tool_trickle_mutex );
162                 ldap_pvt_thread_cond_signal( &bdb_tool_trickle_cond );
163                 ldap_pvt_thread_mutex_unlock( &bdb_tool_trickle_mutex );
164 #endif
165                 ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex );
166                 bdb_tool_index_tcount = slap_tool_thread_max - 1;
167                 ldap_pvt_thread_cond_broadcast( &bdb_tool_index_cond_work );
168                 ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex );
169         }
170
171         if( eh.bv.bv_val ) {
172                 ch_free( eh.bv.bv_val );
173                 eh.bv.bv_val = NULL;
174         }
175
176         if( cursor ) {
177                 cursor->c_close( cursor );
178                 cursor = NULL;
179         }
180
181 #ifdef BDB_TOOL_IDL_CACHING
182         bdb_tool_idl_flush( be );
183 #endif
184
185         if( nholes ) {
186                 unsigned i;
187                 fprintf( stderr, "Error, entries missing!\n");
188                 for (i=0; i<nholes; i++) {
189                         fprintf(stderr, "  entry %ld: %s\n",
190                                 holes[i].id, holes[i].dn.bv_val);
191                 }
192                 return -1;
193         }
194                         
195         return 0;
196 }
197
198 ID
199 bdb_tool_entry_first_x(
200         BackendDB *be,
201         struct berval *base,
202         int scope,
203         Filter *f )
204 {
205         tool_base = base;
206         tool_scope = scope;
207         tool_filter = f;
208         
209         return bdb_tool_entry_next( be );
210 }
211
212 ID bdb_tool_entry_next(
213         BackendDB *be )
214 {
215         int rc;
216         ID id;
217         struct bdb_info *bdb = (struct bdb_info *) be->be_private;
218
219         assert( be != NULL );
220         assert( slapMode & SLAP_TOOL_MODE );
221         assert( bdb != NULL );
222
223 next:;
224         /* Get the header */
225         data.ulen = data.dlen = sizeof( ehbuf );
226         data.data = ehbuf;
227         data.flags |= DB_DBT_PARTIAL;
228         rc = cursor->c_get( cursor, &key, &data, DB_NEXT );
229
230         if( rc ) {
231                 /* If we're doing linear indexing and there are more attrs to
232                  * index, and we're at the end of the database, start over.
233                  */
234                 if ( index_nattrs && rc == DB_NOTFOUND ) {
235                         /* optional - do a checkpoint here? */
236                         bdb_attr_info_free( bdb->bi_attrs[0] );
237                         bdb->bi_attrs[0] = bdb->bi_attrs[index_nattrs];
238                         index_nattrs--;
239                         rc = cursor->c_get( cursor, &key, &data, DB_FIRST );
240                         if ( rc ) {
241                                 return NOID;
242                         }
243                 } else {
244                         return NOID;
245                 }
246         }
247
248         BDB_DISK2ID( key.data, &id );
249         previd = id;
250
251         if ( tool_filter || tool_base ) {
252                 static Operation op = {0};
253                 static Opheader ohdr = {0};
254
255                 op.o_hdr = &ohdr;
256                 op.o_bd = be;
257                 op.o_tmpmemctx = NULL;
258                 op.o_tmpmfuncs = &ch_mfuncs;
259
260                 if ( tool_next_entry ) {
261                         bdb_entry_release( &op, tool_next_entry, 0 );
262                         tool_next_entry = NULL;
263                 }
264
265                 rc = bdb_tool_entry_get_int( be, id, &tool_next_entry );
266                 if ( rc == LDAP_NO_SUCH_OBJECT ) {
267                         goto next;
268                 }
269
270                 assert( tool_next_entry != NULL );
271
272 #ifdef BDB_HIER
273                 /* TODO: needed until BDB_HIER is handled accordingly
274                  * in bdb_tool_entry_get_int() */
275                 if ( tool_base && !dnIsSuffixScope( &tool_next_entry->e_nname, tool_base, tool_scope ) )
276                 {
277                         bdb_entry_release( &op, tool_next_entry, 0 );
278                         tool_next_entry = NULL;
279                         goto next;
280                 }
281 #endif
282
283                 if ( tool_filter && test_filter( NULL, tool_next_entry, tool_filter ) != LDAP_COMPARE_TRUE )
284                 {
285                         bdb_entry_release( &op, tool_next_entry, 0 );
286                         tool_next_entry = NULL;
287                         goto next;
288                 }
289         }
290
291         return id;
292 }
293
294 ID bdb_tool_dn2id_get(
295         Backend *be,
296         struct berval *dn
297 )
298 {
299         Operation op = {0};
300         Opheader ohdr = {0};
301         EntryInfo *ei = NULL;
302         int rc;
303
304         if ( BER_BVISEMPTY(dn) )
305                 return 0;
306
307         op.o_hdr = &ohdr;
308         op.o_bd = be;
309         op.o_tmpmemctx = NULL;
310         op.o_tmpmfuncs = &ch_mfuncs;
311
312         rc = bdb_cache_find_ndn( &op, 0, dn, &ei );
313         if ( ei ) bdb_cache_entryinfo_unlock( ei );
314         if ( rc == DB_NOTFOUND )
315                 return NOID;
316         
317         return ei->bei_id;
318 }
319
320 static int
321 bdb_tool_entry_get_int( BackendDB *be, ID id, Entry **ep )
322 {
323         Entry *e = NULL;
324         char *dptr;
325         int rc, eoff;
326
327         assert( be != NULL );
328         assert( slapMode & SLAP_TOOL_MODE );
329
330         if ( ( tool_filter || tool_base ) && id == previd && tool_next_entry != NULL ) {
331                 *ep = tool_next_entry;
332                 tool_next_entry = NULL;
333                 return LDAP_SUCCESS;
334         }
335
336         if ( id != previd ) {
337                 data.ulen = data.dlen = sizeof( ehbuf );
338                 data.data = ehbuf;
339                 data.flags |= DB_DBT_PARTIAL;
340
341                 BDB_ID2DISK( id, &nid );
342                 rc = cursor->c_get( cursor, &key, &data, DB_SET );
343                 if ( rc ) {
344                         rc = LDAP_OTHER;
345                         goto done;
346                 }
347         }
348
349         /* Get the header */
350         dptr = eh.bv.bv_val;
351         eh.bv.bv_val = ehbuf;
352         eh.bv.bv_len = data.size;
353         rc = entry_header( &eh );
354         eoff = eh.data - eh.bv.bv_val;
355         eh.bv.bv_val = dptr;
356         if ( rc ) {
357                 rc = LDAP_OTHER;
358                 goto done;
359         }
360
361         /* Get the size */
362         data.flags &= ~DB_DBT_PARTIAL;
363         data.ulen = 0;
364         rc = cursor->c_get( cursor, &key, &data, DB_CURRENT );
365         if ( rc != DB_BUFFER_SMALL ) {
366                 rc = LDAP_OTHER;
367                 goto done;
368         }
369
370         /* Allocate a block and retrieve the data */
371         eh.bv.bv_len = eh.nvals * sizeof( struct berval ) + data.size;
372         eh.bv.bv_val = ch_realloc( eh.bv.bv_val, eh.bv.bv_len );
373         eh.data = eh.bv.bv_val + eh.nvals * sizeof( struct berval );
374         data.data = eh.data;
375         data.ulen = data.size;
376
377         /* Skip past already parsed nattr/nvals */
378         eh.data += eoff;
379
380         rc = cursor->c_get( cursor, &key, &data, DB_CURRENT );
381         if ( rc ) {
382                 rc = LDAP_OTHER;
383                 goto done;
384         }
385
386 #ifndef BDB_HIER
387         /* TODO: handle BDB_HIER accordingly */
388         if ( tool_base != NULL ) {
389                 struct berval ndn;
390                 entry_decode_dn( &eh, NULL, &ndn );
391
392                 if ( !dnIsSuffixScope( &ndn, tool_base, tool_scope ) ) {
393                         return LDAP_NO_SUCH_OBJECT;
394                 }
395         }
396 #endif
397
398 #ifdef SLAP_ZONE_ALLOC
399         /* FIXME: will add ctx later */
400         rc = entry_decode( &eh, &e, NULL );
401 #else
402         rc = entry_decode( &eh, &e );
403 #endif
404
405         if( rc == LDAP_SUCCESS ) {
406                 e->e_id = id;
407 #ifdef BDB_HIER
408                 if ( slapMode & SLAP_TOOL_READONLY ) {
409                         struct bdb_info *bdb = (struct bdb_info *) be->be_private;
410                         EntryInfo *ei = NULL;
411                         Operation op = {0};
412                         Opheader ohdr = {0};
413
414                         op.o_hdr = &ohdr;
415                         op.o_bd = be;
416                         op.o_tmpmemctx = NULL;
417                         op.o_tmpmfuncs = &ch_mfuncs;
418
419                         rc = bdb_cache_find_parent( &op, bdb->bi_cache.c_txn, id, &ei );
420                         if ( rc == LDAP_SUCCESS ) {
421                                 bdb_cache_entryinfo_unlock( ei );
422                                 e->e_private = ei;
423                                 ei->bei_e = e;
424                                 bdb_fix_dn( e, 0 );
425                                 ei->bei_e = NULL;
426                                 e->e_private = NULL;
427                         }
428                 }
429 #endif
430         }
431 done:
432         if ( e != NULL ) {
433                 *ep = e;
434         }
435
436         return rc;
437 }
438
439 Entry*
440 bdb_tool_entry_get( BackendDB *be, ID id )
441 {
442         Entry *e = NULL;
443
444         (void)bdb_tool_entry_get_int( be, id, &e );
445         return e;
446 }
447
448 static int bdb_tool_next_id(
449         Operation *op,
450         DB_TXN *tid,
451         Entry *e,
452         struct berval *text,
453         int hole )
454 {
455         struct berval dn = e->e_name;
456         struct berval ndn = e->e_nname;
457         struct berval pdn, npdn;
458         EntryInfo *ei = NULL, eidummy;
459         int rc;
460
461         if (ndn.bv_len == 0) {
462                 e->e_id = 0;
463                 return 0;
464         }
465
466         rc = bdb_cache_find_ndn( op, tid, &ndn, &ei );
467         if ( ei ) bdb_cache_entryinfo_unlock( ei );
468         if ( rc == DB_NOTFOUND ) {
469                 if ( !be_issuffix( op->o_bd, &ndn ) ) {
470                         ID eid = e->e_id;
471                         dnParent( &dn, &pdn );
472                         dnParent( &ndn, &npdn );
473                         e->e_name = pdn;
474                         e->e_nname = npdn;
475                         rc = bdb_tool_next_id( op, tid, e, text, 1 );
476                         e->e_name = dn;
477                         e->e_nname = ndn;
478                         if ( rc ) {
479                                 return rc;
480                         }
481                         /* If parent didn't exist, it was created just now
482                          * and its ID is now in e->e_id. Make sure the current
483                          * entry gets added under the new parent ID.
484                          */
485                         if ( eid != e->e_id ) {
486                                 eidummy.bei_id = e->e_id;
487                                 ei = &eidummy;
488                         }
489                 }
490                 rc = bdb_next_id( op->o_bd, &e->e_id );
491                 if ( rc ) {
492                         snprintf( text->bv_val, text->bv_len,
493                                 "next_id failed: %s (%d)",
494                                 db_strerror(rc), rc );
495                 Debug( LDAP_DEBUG_ANY,
496                         "=> bdb_tool_next_id: %s\n", text->bv_val, 0, 0 );
497                         return rc;
498                 }
499                 rc = bdb_dn2id_add( op, tid, ei, e );
500                 if ( rc ) {
501                         snprintf( text->bv_val, text->bv_len, 
502                                 "dn2id_add failed: %s (%d)",
503                                 db_strerror(rc), rc );
504                 Debug( LDAP_DEBUG_ANY,
505                         "=> bdb_tool_next_id: %s\n", text->bv_val, 0, 0 );
506                 } else if ( hole ) {
507                         if ( nholes == nhmax - 1 ) {
508                                 if ( holes == hbuf ) {
509                                         holes = ch_malloc( nhmax * sizeof(dn_id) * 2 );
510                                         AC_MEMCPY( holes, hbuf, sizeof(hbuf) );
511                                 } else {
512                                         holes = ch_realloc( holes, nhmax * sizeof(dn_id) * 2 );
513                                 }
514                                 nhmax *= 2;
515                         }
516                         ber_dupbv( &holes[nholes].dn, &ndn );
517                         holes[nholes++].id = e->e_id;
518                 }
519         } else if ( !hole ) {
520                 unsigned i, j;
521
522                 e->e_id = ei->bei_id;
523
524                 for ( i=0; i<nholes; i++) {
525                         if ( holes[i].id == e->e_id ) {
526                                 free(holes[i].dn.bv_val);
527                                 for (j=i;j<nholes;j++) holes[j] = holes[j+1];
528                                 holes[j].id = 0;
529                                 nholes--;
530                                 break;
531                         } else if ( holes[i].id > e->e_id ) {
532                                 break;
533                         }
534                 }
535         }
536         return rc;
537 }
538
539 static int
540 bdb_tool_index_add(
541         Operation *op,
542         DB_TXN *txn,
543         Entry *e )
544 {
545         struct bdb_info *bdb = (struct bdb_info *) op->o_bd->be_private;
546
547         if ( !bdb->bi_nattrs )
548                 return 0;
549
550         if ( slapMode & SLAP_TOOL_QUICK ) {
551                 IndexRec *ir;
552                 int i, rc;
553                 Attribute *a;
554                 
555                 ir = bdb_tool_index_rec;
556                 memset(ir, 0, bdb->bi_nattrs * sizeof( IndexRec ));
557
558                 for ( a = e->e_attrs; a != NULL; a = a->a_next ) {
559                         rc = bdb_index_recset( bdb, a, a->a_desc->ad_type, 
560                                 &a->a_desc->ad_tags, ir );
561                         if ( rc )
562                                 return rc;
563                 }
564                 bdb_tool_ix_id = e->e_id;
565                 bdb_tool_ix_op = op;
566                 ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex );
567                 /* Wait for all threads to be ready */
568                 while ( bdb_tool_index_tcount ) {
569                         ldap_pvt_thread_cond_wait( &bdb_tool_index_cond_main, 
570                                 &bdb_tool_index_mutex );
571                 }
572                 for ( i=1; i<slap_tool_thread_max; i++ )
573                         bdb_tool_index_threads[i] = LDAP_BUSY;
574                 bdb_tool_index_tcount = slap_tool_thread_max - 1;
575                 ldap_pvt_thread_cond_broadcast( &bdb_tool_index_cond_work );
576                 ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex );
577                 rc = bdb_index_recrun( op, bdb, ir, e->e_id, 0 );
578                 if ( rc )
579                         return rc;
580                 ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex );
581                 for ( i=1; i<slap_tool_thread_max; i++ ) {
582                         if ( bdb_tool_index_threads[i] == LDAP_BUSY ) {
583                                 ldap_pvt_thread_cond_wait( &bdb_tool_index_cond_main, 
584                                         &bdb_tool_index_mutex );
585                                 i--;
586                                 continue;
587                         }
588                         if ( bdb_tool_index_threads[i] ) {
589                                 rc = bdb_tool_index_threads[i];
590                                 break;
591                         }
592                 }
593                 ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex );
594                 return rc;
595         } else {
596                 return bdb_index_entry_add( op, txn, e );
597         }
598 }
599
600 ID bdb_tool_entry_put(
601         BackendDB *be,
602         Entry *e,
603         struct berval *text )
604 {
605         int rc;
606         struct bdb_info *bdb = (struct bdb_info *) be->be_private;
607         DB_TXN *tid = NULL;
608         Operation op = {0};
609         Opheader ohdr = {0};
610
611         assert( be != NULL );
612         assert( slapMode & SLAP_TOOL_MODE );
613
614         assert( text != NULL );
615         assert( text->bv_val != NULL );
616         assert( text->bv_val[0] == '\0' );      /* overconservative? */
617
618         Debug( LDAP_DEBUG_TRACE, "=> " LDAP_XSTRING(bdb_tool_entry_put)
619                 "( %ld, \"%s\" )\n", (long) e->e_id, e->e_dn, 0 );
620
621         if (! (slapMode & SLAP_TOOL_QUICK)) {
622         rc = TXN_BEGIN( bdb->bi_dbenv, NULL, &tid, 
623                 bdb->bi_db_opflags );
624         if( rc != 0 ) {
625                 snprintf( text->bv_val, text->bv_len,
626                         "txn_begin failed: %s (%d)",
627                         db_strerror(rc), rc );
628                 Debug( LDAP_DEBUG_ANY,
629                         "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
630                          text->bv_val, 0, 0 );
631                 return NOID;
632         }
633         }
634
635         op.o_hdr = &ohdr;
636         op.o_bd = be;
637         op.o_tmpmemctx = NULL;
638         op.o_tmpmfuncs = &ch_mfuncs;
639
640         /* add dn2id indices */
641         rc = bdb_tool_next_id( &op, tid, e, text, 0 );
642         if( rc != 0 ) {
643                 goto done;
644         }
645
646 #ifdef USE_TRICKLE
647         if (( slapMode & SLAP_TOOL_QUICK ) && (( e->e_id & 0xfff ) == 0xfff )) {
648                 ldap_pvt_thread_cond_signal( &bdb_tool_trickle_cond );
649         }
650 #endif
651
652         if ( !bdb->bi_linear_index )
653                 rc = bdb_tool_index_add( &op, tid, e );
654         if( rc != 0 ) {
655                 snprintf( text->bv_val, text->bv_len,
656                                 "index_entry_add failed: %s (%d)",
657                                 rc == LDAP_OTHER ? "Internal error" :
658                                 db_strerror(rc), rc );
659                 Debug( LDAP_DEBUG_ANY,
660                         "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
661                         text->bv_val, 0, 0 );
662                 goto done;
663         }
664
665         /* id2entry index */
666         rc = bdb_id2entry_add( be, tid, e );
667         if( rc != 0 ) {
668                 snprintf( text->bv_val, text->bv_len,
669                                 "id2entry_add failed: %s (%d)",
670                                 db_strerror(rc), rc );
671                 Debug( LDAP_DEBUG_ANY,
672                         "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
673                         text->bv_val, 0, 0 );
674                 goto done;
675         }
676
677 done:
678         if( rc == 0 ) {
679                 if ( !( slapMode & SLAP_TOOL_QUICK )) {
680                 rc = TXN_COMMIT( tid, 0 );
681                 if( rc != 0 ) {
682                         snprintf( text->bv_val, text->bv_len,
683                                         "txn_commit failed: %s (%d)",
684                                         db_strerror(rc), rc );
685                         Debug( LDAP_DEBUG_ANY,
686                                 "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
687                                 text->bv_val, 0, 0 );
688                         e->e_id = NOID;
689                 }
690                 }
691
692         } else {
693                 if ( !( slapMode & SLAP_TOOL_QUICK )) {
694                 TXN_ABORT( tid );
695                 snprintf( text->bv_val, text->bv_len,
696                         "txn_aborted! %s (%d)",
697                         rc == LDAP_OTHER ? "Internal error" :
698                         db_strerror(rc), rc );
699                 Debug( LDAP_DEBUG_ANY,
700                         "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
701                         text->bv_val, 0, 0 );
702                 }
703                 e->e_id = NOID;
704         }
705
706         return e->e_id;
707 }
708
709 int bdb_tool_entry_reindex(
710         BackendDB *be,
711         ID id,
712         AttributeDescription **adv )
713 {
714         struct bdb_info *bi = (struct bdb_info *) be->be_private;
715         int rc;
716         Entry *e;
717         DB_TXN *tid = NULL;
718         Operation op = {0};
719         Opheader ohdr = {0};
720
721         assert( tool_base == NULL );
722         assert( tool_filter == NULL );
723
724         Debug( LDAP_DEBUG_ARGS,
725                 "=> " LDAP_XSTRING(bdb_tool_entry_reindex) "( %ld )\n",
726                 (long) id, 0, 0 );
727
728         /* No indexes configured, nothing to do. Could return an
729          * error here to shortcut things.
730          */
731         if (!bi->bi_attrs) {
732                 return 0;
733         }
734
735         /* Check for explicit list of attrs to index */
736         if ( adv ) {
737                 int i, j, n;
738
739                 if ( bi->bi_attrs[0]->ai_desc != adv[0] ) {
740                         /* count */
741                         for ( n = 0; adv[n]; n++ ) ;
742
743                         /* insertion sort */
744                         for ( i = 0; i < n; i++ ) {
745                                 AttributeDescription *ad = adv[i];
746                                 for ( j = i-1; j>=0; j--) {
747                                         if ( SLAP_PTRCMP( adv[j], ad ) <= 0 ) break;
748                                         adv[j+1] = adv[j];
749                                 }
750                                 adv[j+1] = ad;
751                         }
752                 }
753
754                 for ( i = 0; adv[i]; i++ ) {
755                         if ( bi->bi_attrs[i]->ai_desc != adv[i] ) {
756                                 for ( j = i+1; j < bi->bi_nattrs; j++ ) {
757                                         if ( bi->bi_attrs[j]->ai_desc == adv[i] ) {
758                                                 AttrInfo *ai = bi->bi_attrs[i];
759                                                 bi->bi_attrs[i] = bi->bi_attrs[j];
760                                                 bi->bi_attrs[j] = ai;
761                                                 break;
762                                         }
763                                 }
764                                 if ( j == bi->bi_nattrs ) {
765                                         Debug( LDAP_DEBUG_ANY,
766                                                 LDAP_XSTRING(bdb_tool_entry_reindex)
767                                                 ": no index configured for %s\n",
768                                                 adv[i]->ad_cname.bv_val, 0, 0 );
769                                         return -1;
770                                 }
771                         }
772                 }
773                 bi->bi_nattrs = i;
774         }
775
776         /* Get the first attribute to index */
777         if (bi->bi_linear_index && !index_nattrs) {
778                 index_nattrs = bi->bi_nattrs - 1;
779                 bi->bi_nattrs = 1;
780         }
781
782         e = bdb_tool_entry_get( be, id );
783
784         if( e == NULL ) {
785                 Debug( LDAP_DEBUG_ANY,
786                         LDAP_XSTRING(bdb_tool_entry_reindex)
787                         ": could not locate id=%ld\n",
788                         (long) id, 0, 0 );
789                 return -1;
790         }
791
792         if (! (slapMode & SLAP_TOOL_QUICK)) {
793         rc = TXN_BEGIN( bi->bi_dbenv, NULL, &tid, bi->bi_db_opflags );
794         if( rc != 0 ) {
795                 Debug( LDAP_DEBUG_ANY,
796                         "=> " LDAP_XSTRING(bdb_tool_entry_reindex) ": "
797                         "txn_begin failed: %s (%d)\n",
798                         db_strerror(rc), rc, 0 );
799                 goto done;
800         }
801         }
802         
803         /*
804          * just (re)add them for now
805          * assume that some other routine (not yet implemented)
806          * will zap index databases
807          *
808          */
809
810         Debug( LDAP_DEBUG_TRACE,
811                 "=> " LDAP_XSTRING(bdb_tool_entry_reindex) "( %ld, \"%s\" )\n",
812                 (long) id, e->e_dn, 0 );
813
814         op.o_hdr = &ohdr;
815         op.o_bd = be;
816         op.o_tmpmemctx = NULL;
817         op.o_tmpmfuncs = &ch_mfuncs;
818
819         rc = bdb_tool_index_add( &op, tid, e );
820
821 done:
822         if( rc == 0 ) {
823                 if (! (slapMode & SLAP_TOOL_QUICK)) {
824                 rc = TXN_COMMIT( tid, 0 );
825                 if( rc != 0 ) {
826                         Debug( LDAP_DEBUG_ANY,
827                                 "=> " LDAP_XSTRING(bdb_tool_entry_reindex)
828                                 ": txn_commit failed: %s (%d)\n",
829                                 db_strerror(rc), rc, 0 );
830                         e->e_id = NOID;
831                 }
832                 }
833
834         } else {
835                 if (! (slapMode & SLAP_TOOL_QUICK)) {
836                 TXN_ABORT( tid );
837                 Debug( LDAP_DEBUG_ANY,
838                         "=> " LDAP_XSTRING(bdb_tool_entry_reindex)
839                         ": txn_aborted! %s (%d)\n",
840                         db_strerror(rc), rc, 0 );
841                 }
842                 e->e_id = NOID;
843         }
844         bdb_entry_release( &op, e, 0 );
845
846         return rc;
847 }
848
849 ID bdb_tool_entry_modify(
850         BackendDB *be,
851         Entry *e,
852         struct berval *text )
853 {
854         int rc;
855         struct bdb_info *bdb = (struct bdb_info *) be->be_private;
856         DB_TXN *tid = NULL;
857         Operation op = {0};
858         Opheader ohdr = {0};
859
860         assert( be != NULL );
861         assert( slapMode & SLAP_TOOL_MODE );
862
863         assert( text != NULL );
864         assert( text->bv_val != NULL );
865         assert( text->bv_val[0] == '\0' );      /* overconservative? */
866
867         assert ( e->e_id != NOID );
868
869         Debug( LDAP_DEBUG_TRACE,
870                 "=> " LDAP_XSTRING(bdb_tool_entry_modify) "( %ld, \"%s\" )\n",
871                 (long) e->e_id, e->e_dn, 0 );
872
873         if (! (slapMode & SLAP_TOOL_QUICK)) {
874                 if( cursor ) {
875                         cursor->c_close( cursor );
876                         cursor = NULL;
877                 }
878                 rc = TXN_BEGIN( bdb->bi_dbenv, NULL, &tid, 
879                         bdb->bi_db_opflags );
880                 if( rc != 0 ) {
881                         snprintf( text->bv_val, text->bv_len,
882                                 "txn_begin failed: %s (%d)",
883                                 db_strerror(rc), rc );
884                         Debug( LDAP_DEBUG_ANY,
885                                 "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n",
886                                  text->bv_val, 0, 0 );
887                         return NOID;
888                 }
889         }
890
891         op.o_hdr = &ohdr;
892         op.o_bd = be;
893         op.o_tmpmemctx = NULL;
894         op.o_tmpmfuncs = &ch_mfuncs;
895
896         /* id2entry index */
897         rc = bdb_id2entry_update( be, tid, e );
898         if( rc != 0 ) {
899                 snprintf( text->bv_val, text->bv_len,
900                                 "id2entry_add failed: %s (%d)",
901                                 db_strerror(rc), rc );
902                 Debug( LDAP_DEBUG_ANY,
903                         "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n",
904                         text->bv_val, 0, 0 );
905                 goto done;
906         }
907
908 done:
909         if( rc == 0 ) {
910                 if (! (slapMode & SLAP_TOOL_QUICK)) {
911                 rc = TXN_COMMIT( tid, 0 );
912                 if( rc != 0 ) {
913                         snprintf( text->bv_val, text->bv_len,
914                                         "txn_commit failed: %s (%d)",
915                                         db_strerror(rc), rc );
916                         Debug( LDAP_DEBUG_ANY,
917                                 "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": "
918                                 "%s\n", text->bv_val, 0, 0 );
919                         e->e_id = NOID;
920                 }
921                 }
922
923         } else {
924                 if (! (slapMode & SLAP_TOOL_QUICK)) {
925                 TXN_ABORT( tid );
926                 snprintf( text->bv_val, text->bv_len,
927                         "txn_aborted! %s (%d)",
928                         db_strerror(rc), rc );
929                 Debug( LDAP_DEBUG_ANY,
930                         "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n",
931                         text->bv_val, 0, 0 );
932                 }
933                 e->e_id = NOID;
934         }
935
936         return e->e_id;
937 }
938
939 #ifdef BDB_TOOL_IDL_CACHING
940 static int
941 bdb_tool_idl_cmp( const void *v1, const void *v2 )
942 {
943         const bdb_tool_idl_cache *c1 = v1, *c2 = v2;
944         int rc;
945
946         if (( rc = c1->kstr.bv_len - c2->kstr.bv_len )) return rc;
947         return memcmp( c1->kstr.bv_val, c2->kstr.bv_val, c1->kstr.bv_len );
948 }
949
950 static int
951 bdb_tool_idl_flush_one( void *v1, void *arg )
952 {
953         bdb_tool_idl_cache *ic = v1;
954         DB *db = arg;
955         struct bdb_info *bdb = bdb_tool_info;
956         bdb_tool_idl_cache_entry *ice;
957         DBC *curs;
958         DBT key, data;
959         int i, rc;
960         ID id, nid;
961
962         /* Freshly allocated, ignore it */
963         if ( !ic->head && ic->count <= BDB_IDL_DB_SIZE ) {
964                 return 0;
965         }
966
967         rc = db->cursor( db, NULL, &curs, 0 );
968         if ( rc )
969                 return -1;
970
971         DBTzero( &key );
972         DBTzero( &data );
973
974         bv2DBT( &ic->kstr, &key );
975
976         data.size = data.ulen = sizeof( ID );
977         data.flags = DB_DBT_USERMEM;
978         data.data = &nid;
979
980         rc = curs->c_get( curs, &key, &data, DB_SET );
981         /* If key already exists and we're writing a range... */
982         if ( rc == 0 && ic->count > BDB_IDL_DB_SIZE ) {
983                 /* If it's not currently a range, must delete old info */
984                 if ( nid ) {
985                         /* Skip lo */
986                         while ( curs->c_get( curs, &key, &data, DB_NEXT_DUP ) == 0 )
987                                 curs->c_del( curs, 0 );
988
989                         nid = 0;
990                         /* Store range marker */
991                         curs->c_put( curs, &key, &data, DB_KEYFIRST );
992                 } else {
993                         
994                         /* Skip lo */
995                         rc = curs->c_get( curs, &key, &data, DB_NEXT_DUP );
996
997                         /* Get hi */
998                         rc = curs->c_get( curs, &key, &data, DB_NEXT_DUP );
999
1000                         /* Delete hi */
1001                         curs->c_del( curs, 0 );
1002                 }
1003                 BDB_ID2DISK( ic->last, &nid );
1004                 curs->c_put( curs, &key, &data, DB_KEYLAST );
1005                 rc = 0;
1006         } else if ( rc && rc != DB_NOTFOUND ) {
1007                 rc = -1;
1008         } else if ( ic->count > BDB_IDL_DB_SIZE ) {
1009                 /* range, didn't exist before */
1010                 nid = 0;
1011                 rc = curs->c_put( curs, &key, &data, DB_KEYLAST );
1012                 if ( rc == 0 ) {
1013                         BDB_ID2DISK( ic->first, &nid );
1014                         rc = curs->c_put( curs, &key, &data, DB_KEYLAST );
1015                         if ( rc == 0 ) {
1016                                 BDB_ID2DISK( ic->last, &nid );
1017                                 rc = curs->c_put( curs, &key, &data, DB_KEYLAST );
1018                         }
1019                 }
1020                 if ( rc ) {
1021                         rc = -1;
1022                 }
1023         } else {
1024                 int n;
1025
1026                 /* Just a normal write */
1027                 rc = 0;
1028                 for ( ice = ic->head, n=0; ice; ice = ice->next, n++ ) {
1029                         int end;
1030                         if ( ice->next ) {
1031                                 end = IDBLOCK;
1032                         } else {
1033                                 end = ic->count & (IDBLOCK-1);
1034                                 if ( !end )
1035                                         end = IDBLOCK;
1036                         }
1037                         for ( i=0; i<end; i++ ) {
1038                                 if ( !ice->ids[i] ) continue;
1039                                 BDB_ID2DISK( ice->ids[i], &nid );
1040                                 rc = curs->c_put( curs, &key, &data, DB_NODUPDATA );
1041                                 if ( rc ) {
1042                                         if ( rc == DB_KEYEXIST ) {
1043                                                 rc = 0;
1044                                                 continue;
1045                                         }
1046                                         rc = -1;
1047                                         break;
1048                                 }
1049                         }
1050                         if ( rc ) {
1051                                 rc = -1;
1052                                 break;
1053                         }
1054                 }
1055                 if ( ic->head ) {
1056                         ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
1057                         ic->tail->next = bdb_tool_idl_free_list;
1058                         bdb_tool_idl_free_list = ic->head;
1059                         bdb->bi_idl_cache_size -= n;
1060                         ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
1061                 }
1062         }
1063         if ( ic != db->app_private ) {
1064                 ch_free( ic );
1065         } else {
1066                 ic->head = ic->tail = NULL;
1067         }
1068         curs->c_close( curs );
1069         return rc;
1070 }
1071
1072 static int
1073 bdb_tool_idl_flush_db( DB *db, bdb_tool_idl_cache *ic )
1074 {
1075         Avlnode *root = db->app_private;
1076         int rc;
1077
1078         db->app_private = ic;
1079         rc = avl_apply( root, bdb_tool_idl_flush_one, db, -1, AVL_INORDER );
1080         avl_free( root, NULL );
1081         db->app_private = NULL;
1082         if ( rc != -1 )
1083                 rc = 0;
1084         return rc;
1085 }
1086
1087 static int
1088 bdb_tool_idl_flush( BackendDB *be )
1089 {
1090         struct bdb_info *bdb = (struct bdb_info *) be->be_private;
1091         DB *db;
1092         Avlnode *root;
1093         int i, rc = 0;
1094
1095         for ( i=BDB_NDB; i < bdb->bi_ndatabases; i++ ) {
1096                 db = bdb->bi_databases[i]->bdi_db;
1097                 if ( !db->app_private ) continue;
1098                 rc = bdb_tool_idl_flush_db( db, NULL );
1099                 if ( rc )
1100                         break;
1101         }
1102         if ( !rc ) {
1103                 bdb->bi_idl_cache_size = 0;
1104         }
1105         return rc;
1106 }
1107
1108 int bdb_tool_idl_add(
1109         BackendDB *be,
1110         DB *db,
1111         DB_TXN *txn,
1112         DBT *key,
1113         ID id )
1114 {
1115         struct bdb_info *bdb = (struct bdb_info *) be->be_private;
1116         bdb_tool_idl_cache *ic, itmp;
1117         bdb_tool_idl_cache_entry *ice;
1118         int rc;
1119
1120         if ( !bdb->bi_idl_cache_max_size )
1121                 return bdb_idl_insert_key( be, db, txn, key, id );
1122
1123         DBT2bv( key, &itmp.kstr );
1124
1125         ic = avl_find( (Avlnode *)db->app_private, &itmp, bdb_tool_idl_cmp );
1126
1127         /* No entry yet, create one */
1128         if ( !ic ) {
1129                 DBC *curs;
1130                 DBT data;
1131                 ID nid;
1132                 int rc;
1133
1134                 ic = ch_malloc( sizeof( bdb_tool_idl_cache ) + itmp.kstr.bv_len );
1135                 ic->kstr.bv_len = itmp.kstr.bv_len;
1136                 ic->kstr.bv_val = (char *)(ic+1);
1137                 AC_MEMCPY( ic->kstr.bv_val, itmp.kstr.bv_val, ic->kstr.bv_len );
1138                 ic->head = ic->tail = NULL;
1139                 ic->last = 0;
1140                 ic->count = 0;
1141                 avl_insert( (Avlnode **)&db->app_private, ic, bdb_tool_idl_cmp,
1142                         avl_dup_error );
1143
1144                 /* load existing key count here */
1145                 rc = db->cursor( db, NULL, &curs, 0 );
1146                 if ( rc ) return rc;
1147
1148                 data.ulen = sizeof( ID );
1149                 data.flags = DB_DBT_USERMEM;
1150                 data.data = &nid;
1151                 rc = curs->c_get( curs, key, &data, DB_SET );
1152                 if ( rc == 0 ) {
1153                         if ( nid == 0 ) {
1154                                 ic->count = BDB_IDL_DB_SIZE+1;
1155                         } else {
1156                                 db_recno_t count;
1157
1158                                 curs->c_count( curs, &count, 0 );
1159                                 ic->count = count;
1160                                 BDB_DISK2ID( &nid, &ic->first );
1161                         }
1162                 }
1163                 curs->c_close( curs );
1164         }
1165         /* are we a range already? */
1166         if ( ic->count > BDB_IDL_DB_SIZE ) {
1167                 ic->last = id;
1168                 return 0;
1169         /* Are we at the limit, and converting to a range? */
1170         } else if ( ic->count == BDB_IDL_DB_SIZE ) {
1171                 int n;
1172                 for ( ice = ic->head, n=0; ice; ice = ice->next, n++ )
1173                         /* counting */ ;
1174                 if ( n ) {
1175                         ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
1176                         ic->tail->next = bdb_tool_idl_free_list;
1177                         bdb_tool_idl_free_list = ic->head;
1178                         bdb->bi_idl_cache_size -= n;
1179                         ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
1180                 }
1181                 ic->head = ic->tail = NULL;
1182                 ic->last = id;
1183                 ic->count++;
1184                 return 0;
1185         }
1186         /* No free block, create that too */
1187         if ( !ic->tail || ( ic->count & (IDBLOCK-1)) == 0) {
1188                 ice = NULL;
1189                 ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
1190                 if ( bdb->bi_idl_cache_size >= bdb->bi_idl_cache_max_size ) {
1191                         ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
1192                         rc = bdb_tool_idl_flush_db( db, ic );
1193                         if ( rc )
1194                                 return rc;
1195                         avl_insert( (Avlnode **)&db->app_private, ic, bdb_tool_idl_cmp,
1196                                 avl_dup_error );
1197                         ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
1198                 }
1199                 bdb->bi_idl_cache_size++;
1200                 if ( bdb_tool_idl_free_list ) {
1201                         ice = bdb_tool_idl_free_list;
1202                         bdb_tool_idl_free_list = ice->next;
1203                 }
1204                 ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
1205                 if ( !ice ) {
1206                         ice = ch_malloc( sizeof( bdb_tool_idl_cache_entry ));
1207                 }
1208                 memset( ice, 0, sizeof( *ice ));
1209                 if ( !ic->head ) {
1210                         ic->head = ice;
1211                 } else {
1212                         ic->tail->next = ice;
1213                 }
1214                 ic->tail = ice;
1215                 if ( !ic->count )
1216                         ic->first = id;
1217         }
1218         ice = ic->tail;
1219         ice->ids[ ic->count & (IDBLOCK-1) ] = id;
1220         ic->count++;
1221
1222         return 0;
1223 }
1224 #endif
1225
1226 #ifdef USE_TRICKLE
1227 static void *
1228 bdb_tool_trickle_task( void *ctx, void *ptr )
1229 {
1230         DB_ENV *env = ptr;
1231         int wrote;
1232
1233         ldap_pvt_thread_mutex_lock( &bdb_tool_trickle_mutex );
1234         while ( 1 ) {
1235                 ldap_pvt_thread_cond_wait( &bdb_tool_trickle_cond,
1236                         &bdb_tool_trickle_mutex );
1237                 if ( slapd_shutdown )
1238                         break;
1239                 env->memp_trickle( env, 30, &wrote );
1240         }
1241         ldap_pvt_thread_mutex_unlock( &bdb_tool_trickle_mutex );
1242
1243         return NULL;
1244 }
1245 #endif
1246
1247 static void *
1248 bdb_tool_index_task( void *ctx, void *ptr )
1249 {
1250         int base = *(int *)ptr;
1251
1252         free( ptr );
1253         while ( 1 ) {
1254                 ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex );
1255                 bdb_tool_index_tcount--;
1256                 if ( !bdb_tool_index_tcount )
1257                         ldap_pvt_thread_cond_signal( &bdb_tool_index_cond_main );
1258                 ldap_pvt_thread_cond_wait( &bdb_tool_index_cond_work,
1259                         &bdb_tool_index_mutex );
1260                 ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex );
1261                 if ( slapd_shutdown )
1262                         break;
1263
1264                 bdb_tool_index_threads[base] = bdb_index_recrun( bdb_tool_ix_op,
1265                         bdb_tool_info, bdb_tool_index_rec, bdb_tool_ix_id, base );
1266         }
1267
1268         return NULL;
1269 }