]> git.sur5r.net Git - openldap/blob - servers/slapd/back-bdb/tools.c
ITS#6759
[openldap] / servers / slapd / back-bdb / tools.c
1 /* tools.c - tools for slap tools */
2 /* $OpenLDAP$ */
3 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
4  *
5  * Copyright 2000-2011 The OpenLDAP Foundation.
6  * All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted only as authorized by the OpenLDAP
10  * Public License.
11  *
12  * A copy of this license is available in the file LICENSE in the
13  * top-level directory of the distribution or, alternatively, at
14  * <http://www.OpenLDAP.org/license.html>.
15  */
16
17 #include "portable.h"
18
19 #include <stdio.h>
20 #include <ac/string.h>
21 #include <ac/errno.h>
22
23 #define AVL_INTERNAL
24 #include "back-bdb.h"
25 #include "idl.h"
26
27 static DBC *cursor = NULL;
28 static DBT key, data;
29 static EntryHeader eh;
30 static ID nid, previd = NOID;
31 static char ehbuf[16];
32
33 typedef struct dn_id {
34         ID id;
35         struct berval dn;
36 } dn_id;
37
38 #define HOLE_SIZE       4096
39 static dn_id hbuf[HOLE_SIZE], *holes = hbuf;
40 static unsigned nhmax = HOLE_SIZE;
41 static unsigned nholes;
42
43 static int index_nattrs;
44
45 static struct berval    *tool_base;
46 static int              tool_scope;
47 static Filter           *tool_filter;
48 static Entry            *tool_next_entry;
49
50 #ifdef BDB_TOOL_IDL_CACHING
51 #define bdb_tool_idl_cmp                BDB_SYMBOL(tool_idl_cmp)
52 #define bdb_tool_idl_flush_one          BDB_SYMBOL(tool_idl_flush_one)
53 #define bdb_tool_idl_flush              BDB_SYMBOL(tool_idl_flush)
54
55 static int bdb_tool_idl_flush( BackendDB *be );
56
57 #define IDBLOCK 1024
58
59 typedef struct bdb_tool_idl_cache_entry {
60         struct bdb_tool_idl_cache_entry *next;
61         ID ids[IDBLOCK];
62 } bdb_tool_idl_cache_entry;
63  
64 typedef struct bdb_tool_idl_cache {
65         struct berval kstr;
66         bdb_tool_idl_cache_entry *head, *tail;
67         ID first, last;
68         int count;
69 } bdb_tool_idl_cache;
70
71 static bdb_tool_idl_cache_entry *bdb_tool_idl_free_list;
72 #endif  /* BDB_TOOL_IDL_CACHING */
73
74 static ID bdb_tool_ix_id;
75 static Operation *bdb_tool_ix_op;
76 static int *bdb_tool_index_threads, bdb_tool_index_tcount;
77 static void *bdb_tool_index_rec;
78 static struct bdb_info *bdb_tool_info;
79 static ldap_pvt_thread_mutex_t bdb_tool_index_mutex;
80 static ldap_pvt_thread_cond_t bdb_tool_index_cond_main;
81 static ldap_pvt_thread_cond_t bdb_tool_index_cond_work;
82
83 #if DB_VERSION_FULL >= 0x04060000
84 #define USE_TRICKLE     1
85 #else
86 /* Seems to slow things down too much in BDB 4.5 */
87 #undef USE_TRICKLE
88 #endif
89
90 #ifdef USE_TRICKLE
91 static ldap_pvt_thread_mutex_t bdb_tool_trickle_mutex;
92 static ldap_pvt_thread_cond_t bdb_tool_trickle_cond;
93
94 static void * bdb_tool_trickle_task( void *ctx, void *ptr );
95 #endif
96
97 static void * bdb_tool_index_task( void *ctx, void *ptr );
98
99 static int
100 bdb_tool_entry_get_int( BackendDB *be, ID id, Entry **ep );
101
102 int bdb_tool_entry_open(
103         BackendDB *be, int mode )
104 {
105         struct bdb_info *bdb = (struct bdb_info *) be->be_private;
106
107         /* initialize key and data thangs */
108         DBTzero( &key );
109         DBTzero( &data );
110         key.flags = DB_DBT_USERMEM;
111         key.data = &nid;
112         key.size = key.ulen = sizeof( nid );
113         data.flags = DB_DBT_USERMEM;
114
115         if (cursor == NULL) {
116                 int rc = bdb->bi_id2entry->bdi_db->cursor(
117                         bdb->bi_id2entry->bdi_db, bdb->bi_cache.c_txn, &cursor,
118                         bdb->bi_db_opflags );
119                 if( rc != 0 ) {
120                         return -1;
121                 }
122         }
123
124         /* Set up for threaded slapindex */
125         if (( slapMode & (SLAP_TOOL_QUICK|SLAP_TOOL_READONLY)) == SLAP_TOOL_QUICK ) {
126                 if ( !bdb_tool_info ) {
127 #ifdef USE_TRICKLE
128                         ldap_pvt_thread_mutex_init( &bdb_tool_trickle_mutex );
129                         ldap_pvt_thread_cond_init( &bdb_tool_trickle_cond );
130                         ldap_pvt_thread_pool_submit( &connection_pool, bdb_tool_trickle_task, bdb->bi_dbenv );
131 #endif
132
133                         ldap_pvt_thread_mutex_init( &bdb_tool_index_mutex );
134                         ldap_pvt_thread_cond_init( &bdb_tool_index_cond_main );
135                         ldap_pvt_thread_cond_init( &bdb_tool_index_cond_work );
136                         if ( bdb->bi_nattrs ) {
137                                 int i;
138                                 bdb_tool_index_threads = ch_malloc( slap_tool_thread_max * sizeof( int ));
139                                 bdb_tool_index_rec = ch_malloc( bdb->bi_nattrs * sizeof( IndexRec ));
140                                 bdb_tool_index_tcount = slap_tool_thread_max - 1;
141                                 for (i=1; i<slap_tool_thread_max; i++) {
142                                         int *ptr = ch_malloc( sizeof( int ));
143                                         *ptr = i;
144                                         ldap_pvt_thread_pool_submit( &connection_pool,
145                                                 bdb_tool_index_task, ptr );
146                                 }
147                         }
148                         bdb_tool_info = bdb;
149                 }
150         }
151
152         return 0;
153 }
154
155 int bdb_tool_entry_close(
156         BackendDB *be )
157 {
158         if ( bdb_tool_info ) {
159                 slapd_shutdown = 1;
160 #ifdef USE_TRICKLE
161                 ldap_pvt_thread_mutex_lock( &bdb_tool_trickle_mutex );
162                 ldap_pvt_thread_cond_signal( &bdb_tool_trickle_cond );
163                 ldap_pvt_thread_mutex_unlock( &bdb_tool_trickle_mutex );
164 #endif
165                 ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex );
166                 bdb_tool_index_tcount = slap_tool_thread_max - 1;
167                 ldap_pvt_thread_cond_broadcast( &bdb_tool_index_cond_work );
168                 ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex );
169         }
170
171         if( eh.bv.bv_val ) {
172                 ch_free( eh.bv.bv_val );
173                 eh.bv.bv_val = NULL;
174         }
175
176         if( cursor ) {
177                 cursor->c_close( cursor );
178                 cursor = NULL;
179         }
180
181 #ifdef BDB_TOOL_IDL_CACHING
182         bdb_tool_idl_flush( be );
183 #endif
184
185         if( nholes ) {
186                 unsigned i;
187                 fprintf( stderr, "Error, entries missing!\n");
188                 for (i=0; i<nholes; i++) {
189                         fprintf(stderr, "  entry %ld: %s\n",
190                                 holes[i].id, holes[i].dn.bv_val);
191                 }
192                 return -1;
193         }
194                         
195         return 0;
196 }
197
198 ID
199 bdb_tool_entry_first_x(
200         BackendDB *be,
201         struct berval *base,
202         int scope,
203         Filter *f )
204 {
205         tool_base = base;
206         tool_scope = scope;
207         tool_filter = f;
208         
209         return bdb_tool_entry_next( be );
210 }
211
212 ID bdb_tool_entry_next(
213         BackendDB *be )
214 {
215         int rc;
216         ID id;
217         struct bdb_info *bdb = (struct bdb_info *) be->be_private;
218
219         assert( be != NULL );
220         assert( slapMode & SLAP_TOOL_MODE );
221         assert( bdb != NULL );
222
223 next:;
224         /* Get the header */
225         data.ulen = data.dlen = sizeof( ehbuf );
226         data.data = ehbuf;
227         data.flags |= DB_DBT_PARTIAL;
228         rc = cursor->c_get( cursor, &key, &data, DB_NEXT );
229
230         if( rc ) {
231                 /* If we're doing linear indexing and there are more attrs to
232                  * index, and we're at the end of the database, start over.
233                  */
234                 if ( index_nattrs && rc == DB_NOTFOUND ) {
235                         /* optional - do a checkpoint here? */
236                         bdb_attr_info_free( bdb->bi_attrs[0] );
237                         bdb->bi_attrs[0] = bdb->bi_attrs[index_nattrs];
238                         index_nattrs--;
239                         rc = cursor->c_get( cursor, &key, &data, DB_FIRST );
240                         if ( rc ) {
241                                 return NOID;
242                         }
243                 } else {
244                         return NOID;
245                 }
246         }
247
248         BDB_DISK2ID( key.data, &id );
249         previd = id;
250
251         if ( tool_filter || tool_base ) {
252                 static Operation op = {0};
253                 static Opheader ohdr = {0};
254
255                 op.o_hdr = &ohdr;
256                 op.o_bd = be;
257                 op.o_tmpmemctx = NULL;
258                 op.o_tmpmfuncs = &ch_mfuncs;
259
260                 if ( tool_next_entry ) {
261                         bdb_entry_release( &op, tool_next_entry, 0 );
262                         tool_next_entry = NULL;
263                 }
264
265                 rc = bdb_tool_entry_get_int( be, id, &tool_next_entry );
266                 if ( rc == LDAP_NO_SUCH_OBJECT ) {
267                         goto next;
268                 }
269
270                 assert( tool_next_entry != NULL );
271
272 #ifdef BDB_HIER
273                 /* TODO: needed until BDB_HIER is handled accordingly
274                  * in bdb_tool_entry_get_int() */
275                 if ( tool_base && !dnIsSuffixScope( &tool_next_entry->e_nname, tool_base, tool_scope ) )
276                 {
277                         bdb_entry_release( &op, tool_next_entry, 0 );
278                         tool_next_entry = NULL;
279                         goto next;
280                 }
281 #endif
282
283                 if ( tool_filter && test_filter( NULL, tool_next_entry, tool_filter ) != LDAP_COMPARE_TRUE )
284                 {
285                         bdb_entry_release( &op, tool_next_entry, 0 );
286                         tool_next_entry = NULL;
287                         goto next;
288                 }
289         }
290
291         return id;
292 }
293
294 ID bdb_tool_dn2id_get(
295         Backend *be,
296         struct berval *dn
297 )
298 {
299         Operation op = {0};
300         Opheader ohdr = {0};
301         EntryInfo *ei = NULL;
302         int rc;
303
304         if ( BER_BVISEMPTY(dn) )
305                 return 0;
306
307         op.o_hdr = &ohdr;
308         op.o_bd = be;
309         op.o_tmpmemctx = NULL;
310         op.o_tmpmfuncs = &ch_mfuncs;
311
312         rc = bdb_cache_find_ndn( &op, 0, dn, &ei );
313         if ( ei ) bdb_cache_entryinfo_unlock( ei );
314         if ( rc == DB_NOTFOUND )
315                 return NOID;
316         
317         return ei->bei_id;
318 }
319
320 static int
321 bdb_tool_entry_get_int( BackendDB *be, ID id, Entry **ep )
322 {
323         Entry *e = NULL;
324         char *dptr;
325         int rc, eoff;
326
327         assert( be != NULL );
328         assert( slapMode & SLAP_TOOL_MODE );
329
330         if ( ( tool_filter || tool_base ) && id == previd && tool_next_entry != NULL ) {
331                 *ep = tool_next_entry;
332                 tool_next_entry = NULL;
333                 return LDAP_SUCCESS;
334         }
335
336         if ( id != previd ) {
337                 data.ulen = data.dlen = sizeof( ehbuf );
338                 data.data = ehbuf;
339                 data.flags |= DB_DBT_PARTIAL;
340
341                 BDB_ID2DISK( id, &nid );
342                 rc = cursor->c_get( cursor, &key, &data, DB_SET );
343                 if ( rc ) {
344                         rc = LDAP_OTHER;
345                         goto done;
346                 }
347         }
348
349         /* Get the header */
350         dptr = eh.bv.bv_val;
351         eh.bv.bv_val = ehbuf;
352         eh.bv.bv_len = data.size;
353         rc = entry_header( &eh );
354         eoff = eh.data - eh.bv.bv_val;
355         eh.bv.bv_val = dptr;
356         if ( rc ) {
357                 rc = LDAP_OTHER;
358                 goto done;
359         }
360
361         /* Get the size */
362         data.flags &= ~DB_DBT_PARTIAL;
363         data.ulen = 0;
364         rc = cursor->c_get( cursor, &key, &data, DB_CURRENT );
365         if ( rc != DB_BUFFER_SMALL ) {
366                 rc = LDAP_OTHER;
367                 goto done;
368         }
369
370         /* Allocate a block and retrieve the data */
371         eh.bv.bv_len = eh.nvals * sizeof( struct berval ) + data.size;
372         eh.bv.bv_val = ch_realloc( eh.bv.bv_val, eh.bv.bv_len );
373         eh.data = eh.bv.bv_val + eh.nvals * sizeof( struct berval );
374         data.data = eh.data;
375         data.ulen = data.size;
376
377         /* Skip past already parsed nattr/nvals */
378         eh.data += eoff;
379
380         rc = cursor->c_get( cursor, &key, &data, DB_CURRENT );
381         if ( rc ) {
382                 rc = LDAP_OTHER;
383                 goto done;
384         }
385
386 #ifndef BDB_HIER
387         /* TODO: handle BDB_HIER accordingly */
388         if ( tool_base != NULL ) {
389                 struct berval ndn;
390                 entry_decode_dn( &eh, NULL, &ndn );
391
392                 if ( !dnIsSuffixScope( &ndn, tool_base, tool_scope ) ) {
393                         return LDAP_NO_SUCH_OBJECT;
394                 }
395         }
396 #endif
397
398 #ifdef SLAP_ZONE_ALLOC
399         /* FIXME: will add ctx later */
400         rc = entry_decode( &eh, &e, NULL );
401 #else
402         rc = entry_decode( &eh, &e );
403 #endif
404
405         if( rc == LDAP_SUCCESS ) {
406                 e->e_id = id;
407 #ifdef BDB_HIER
408                 if ( slapMode & SLAP_TOOL_READONLY ) {
409                         struct bdb_info *bdb = (struct bdb_info *) be->be_private;
410                         EntryInfo *ei = NULL;
411                         Operation op = {0};
412                         Opheader ohdr = {0};
413
414                         op.o_hdr = &ohdr;
415                         op.o_bd = be;
416                         op.o_tmpmemctx = NULL;
417                         op.o_tmpmfuncs = &ch_mfuncs;
418
419                         rc = bdb_cache_find_parent( &op, bdb->bi_cache.c_txn, id, &ei );
420                         if ( rc == LDAP_SUCCESS ) {
421                                 bdb_cache_entryinfo_unlock( ei );
422                                 e->e_private = ei;
423                                 ei->bei_e = e;
424                                 bdb_fix_dn( e, 0 );
425                                 ei->bei_e = NULL;
426                                 e->e_private = NULL;
427                         }
428                 }
429 #endif
430         }
431 done:
432         if ( e != NULL ) {
433                 *ep = e;
434         }
435
436         return rc;
437 }
438
439 Entry*
440 bdb_tool_entry_get( BackendDB *be, ID id )
441 {
442         Entry *e = NULL;
443
444         (void)bdb_tool_entry_get_int( be, id, &e );
445         return e;
446 }
447
448 static int bdb_tool_next_id(
449         Operation *op,
450         DB_TXN *tid,
451         Entry *e,
452         struct berval *text,
453         int hole )
454 {
455         struct berval dn = e->e_name;
456         struct berval ndn = e->e_nname;
457         struct berval pdn, npdn;
458         EntryInfo *ei = NULL, eidummy;
459         int rc;
460
461         if (ndn.bv_len == 0) {
462                 e->e_id = 0;
463                 return 0;
464         }
465
466         rc = bdb_cache_find_ndn( op, tid, &ndn, &ei );
467         if ( ei ) bdb_cache_entryinfo_unlock( ei );
468         if ( rc == DB_NOTFOUND ) {
469                 if ( !be_issuffix( op->o_bd, &ndn ) ) {
470                         ID eid = e->e_id;
471                         dnParent( &dn, &pdn );
472                         dnParent( &ndn, &npdn );
473                         e->e_name = pdn;
474                         e->e_nname = npdn;
475                         rc = bdb_tool_next_id( op, tid, e, text, 1 );
476                         e->e_name = dn;
477                         e->e_nname = ndn;
478                         if ( rc ) {
479                                 return rc;
480                         }
481                         /* If parent didn't exist, it was created just now
482                          * and its ID is now in e->e_id. Make sure the current
483                          * entry gets added under the new parent ID.
484                          */
485                         if ( eid != e->e_id ) {
486                                 eidummy.bei_id = e->e_id;
487                                 ei = &eidummy;
488                         }
489                 }
490                 rc = bdb_next_id( op->o_bd, &e->e_id );
491                 if ( rc ) {
492                         snprintf( text->bv_val, text->bv_len,
493                                 "next_id failed: %s (%d)",
494                                 db_strerror(rc), rc );
495                 Debug( LDAP_DEBUG_ANY,
496                         "=> bdb_tool_next_id: %s\n", text->bv_val, 0, 0 );
497                         return rc;
498                 }
499                 rc = bdb_dn2id_add( op, tid, ei, e );
500                 if ( rc ) {
501                         snprintf( text->bv_val, text->bv_len, 
502                                 "dn2id_add failed: %s (%d)",
503                                 db_strerror(rc), rc );
504                 Debug( LDAP_DEBUG_ANY,
505                         "=> bdb_tool_next_id: %s\n", text->bv_val, 0, 0 );
506                 } else if ( hole ) {
507                         if ( nholes == nhmax - 1 ) {
508                                 if ( holes == hbuf ) {
509                                         holes = ch_malloc( nhmax * sizeof(dn_id) * 2 );
510                                         AC_MEMCPY( holes, hbuf, sizeof(hbuf) );
511                                 } else {
512                                         holes = ch_realloc( holes, nhmax * sizeof(dn_id) * 2 );
513                                 }
514                                 nhmax *= 2;
515                         }
516                         ber_dupbv( &holes[nholes].dn, &ndn );
517                         holes[nholes++].id = e->e_id;
518                 }
519         } else if ( !hole ) {
520                 unsigned i, j;
521
522                 e->e_id = ei->bei_id;
523
524                 for ( i=0; i<nholes; i++) {
525                         if ( holes[i].id == e->e_id ) {
526                                 free(holes[i].dn.bv_val);
527                                 for (j=i;j<nholes;j++) holes[j] = holes[j+1];
528                                 holes[j].id = 0;
529                                 nholes--;
530                                 break;
531                         } else if ( holes[i].id > e->e_id ) {
532                                 break;
533                         }
534                 }
535         }
536         return rc;
537 }
538
539 static int
540 bdb_tool_index_add(
541         Operation *op,
542         DB_TXN *txn,
543         Entry *e )
544 {
545         struct bdb_info *bdb = (struct bdb_info *) op->o_bd->be_private;
546
547         if ( !bdb->bi_nattrs )
548                 return 0;
549
550         if ( slapMode & SLAP_TOOL_QUICK ) {
551                 IndexRec *ir;
552                 int i, rc;
553                 Attribute *a;
554                 
555                 ir = bdb_tool_index_rec;
556                 memset(ir, 0, bdb->bi_nattrs * sizeof( IndexRec ));
557
558                 for ( a = e->e_attrs; a != NULL; a = a->a_next ) {
559                         rc = bdb_index_recset( bdb, a, a->a_desc->ad_type, 
560                                 &a->a_desc->ad_tags, ir );
561                         if ( rc )
562                                 return rc;
563                 }
564                 bdb_tool_ix_id = e->e_id;
565                 bdb_tool_ix_op = op;
566                 ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex );
567                 /* Wait for all threads to be ready */
568                 while ( bdb_tool_index_tcount ) {
569                         ldap_pvt_thread_cond_wait( &bdb_tool_index_cond_main, 
570                                 &bdb_tool_index_mutex );
571                 }
572                 for ( i=1; i<slap_tool_thread_max; i++ )
573                         bdb_tool_index_threads[i] = LDAP_BUSY;
574                 bdb_tool_index_tcount = slap_tool_thread_max - 1;
575                 ldap_pvt_thread_cond_broadcast( &bdb_tool_index_cond_work );
576                 ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex );
577                 rc = bdb_index_recrun( op, bdb, ir, e->e_id, 0 );
578                 if ( rc )
579                         return rc;
580                 ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex );
581                 for ( i=1; i<slap_tool_thread_max; i++ ) {
582                         if ( bdb_tool_index_threads[i] == LDAP_BUSY ) {
583                                 ldap_pvt_thread_cond_wait( &bdb_tool_index_cond_main, 
584                                         &bdb_tool_index_mutex );
585                                 i--;
586                                 continue;
587                         }
588                         if ( bdb_tool_index_threads[i] ) {
589                                 rc = bdb_tool_index_threads[i];
590                                 break;
591                         }
592                 }
593                 ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex );
594                 return rc;
595         } else {
596                 return bdb_index_entry_add( op, txn, e );
597         }
598 }
599
600 ID bdb_tool_entry_put(
601         BackendDB *be,
602         Entry *e,
603         struct berval *text )
604 {
605         int rc;
606         struct bdb_info *bdb = (struct bdb_info *) be->be_private;
607         DB_TXN *tid = NULL;
608         Operation op = {0};
609         Opheader ohdr = {0};
610
611         assert( be != NULL );
612         assert( slapMode & SLAP_TOOL_MODE );
613
614         assert( text != NULL );
615         assert( text->bv_val != NULL );
616         assert( text->bv_val[0] == '\0' );      /* overconservative? */
617
618         Debug( LDAP_DEBUG_TRACE, "=> " LDAP_XSTRING(bdb_tool_entry_put)
619                 "( %ld, \"%s\" )\n", (long) e->e_id, e->e_dn, 0 );
620
621         if (! (slapMode & SLAP_TOOL_QUICK)) {
622         rc = TXN_BEGIN( bdb->bi_dbenv, NULL, &tid, 
623                 bdb->bi_db_opflags );
624         if( rc != 0 ) {
625                 snprintf( text->bv_val, text->bv_len,
626                         "txn_begin failed: %s (%d)",
627                         db_strerror(rc), rc );
628                 Debug( LDAP_DEBUG_ANY,
629                         "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
630                          text->bv_val, 0, 0 );
631                 return NOID;
632         }
633         }
634
635         op.o_hdr = &ohdr;
636         op.o_bd = be;
637         op.o_tmpmemctx = NULL;
638         op.o_tmpmfuncs = &ch_mfuncs;
639
640         /* add dn2id indices */
641         rc = bdb_tool_next_id( &op, tid, e, text, 0 );
642         if( rc != 0 ) {
643                 goto done;
644         }
645
646 #ifdef USE_TRICKLE
647         if (( slapMode & SLAP_TOOL_QUICK ) && (( e->e_id & 0xfff ) == 0xfff )) {
648                 ldap_pvt_thread_cond_signal( &bdb_tool_trickle_cond );
649         }
650 #endif
651
652         if ( !bdb->bi_linear_index )
653                 rc = bdb_tool_index_add( &op, tid, e );
654         if( rc != 0 ) {
655                 snprintf( text->bv_val, text->bv_len,
656                                 "index_entry_add failed: %s (%d)",
657                                 rc == LDAP_OTHER ? "Internal error" :
658                                 db_strerror(rc), rc );
659                 Debug( LDAP_DEBUG_ANY,
660                         "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
661                         text->bv_val, 0, 0 );
662                 goto done;
663         }
664
665         /* id2entry index */
666         rc = bdb_id2entry_add( be, tid, e );
667         if( rc != 0 ) {
668                 snprintf( text->bv_val, text->bv_len,
669                                 "id2entry_add failed: %s (%d)",
670                                 db_strerror(rc), rc );
671                 Debug( LDAP_DEBUG_ANY,
672                         "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
673                         text->bv_val, 0, 0 );
674                 goto done;
675         }
676
677 done:
678         if( rc == 0 ) {
679                 if ( !( slapMode & SLAP_TOOL_QUICK )) {
680                 rc = TXN_COMMIT( tid, 0 );
681                 if( rc != 0 ) {
682                         snprintf( text->bv_val, text->bv_len,
683                                         "txn_commit failed: %s (%d)",
684                                         db_strerror(rc), rc );
685                         Debug( LDAP_DEBUG_ANY,
686                                 "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
687                                 text->bv_val, 0, 0 );
688                         e->e_id = NOID;
689                 }
690                 }
691
692         } else {
693                 if ( !( slapMode & SLAP_TOOL_QUICK )) {
694                 TXN_ABORT( tid );
695                 snprintf( text->bv_val, text->bv_len,
696                         "txn_aborted! %s (%d)",
697                         rc == LDAP_OTHER ? "Internal error" :
698                         db_strerror(rc), rc );
699                 Debug( LDAP_DEBUG_ANY,
700                         "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
701                         text->bv_val, 0, 0 );
702                 }
703                 e->e_id = NOID;
704         }
705
706         return e->e_id;
707 }
708
709 int bdb_tool_entry_reindex(
710         BackendDB *be,
711         ID id,
712         AttributeDescription **adv )
713 {
714         struct bdb_info *bi = (struct bdb_info *) be->be_private;
715         int rc;
716         Entry *e;
717         DB_TXN *tid = NULL;
718         Operation op = {0};
719         Opheader ohdr = {0};
720
721         Debug( LDAP_DEBUG_ARGS,
722                 "=> " LDAP_XSTRING(bdb_tool_entry_reindex) "( %ld )\n",
723                 (long) id, 0, 0 );
724         assert( tool_base == NULL );
725         assert( tool_filter == NULL );
726
727         /* No indexes configured, nothing to do. Could return an
728          * error here to shortcut things.
729          */
730         if (!bi->bi_attrs) {
731                 return 0;
732         }
733
734         /* Check for explicit list of attrs to index */
735         if ( adv ) {
736                 int i, j, n;
737
738                 if ( bi->bi_attrs[0]->ai_desc != adv[0] ) {
739                         /* count */
740                         for ( n = 0; adv[n]; n++ ) ;
741
742                         /* insertion sort */
743                         for ( i = 0; i < n; i++ ) {
744                                 AttributeDescription *ad = adv[i];
745                                 for ( j = i-1; j>=0; j--) {
746                                         if ( SLAP_PTRCMP( adv[j], ad ) <= 0 ) break;
747                                         adv[j+1] = adv[j];
748                                 }
749                                 adv[j+1] = ad;
750                         }
751                 }
752
753                 for ( i = 0; adv[i]; i++ ) {
754                         if ( bi->bi_attrs[i]->ai_desc != adv[i] ) {
755                                 for ( j = i+1; j < bi->bi_nattrs; j++ ) {
756                                         if ( bi->bi_attrs[j]->ai_desc == adv[i] ) {
757                                                 AttrInfo *ai = bi->bi_attrs[i];
758                                                 bi->bi_attrs[i] = bi->bi_attrs[j];
759                                                 bi->bi_attrs[j] = ai;
760                                                 break;
761                                         }
762                                 }
763                                 if ( j == bi->bi_nattrs ) {
764                                         Debug( LDAP_DEBUG_ANY,
765                                                 LDAP_XSTRING(bdb_tool_entry_reindex)
766                                                 ": no index configured for %s\n",
767                                                 adv[i]->ad_cname.bv_val, 0, 0 );
768                                         return -1;
769                                 }
770                         }
771                 }
772                 bi->bi_nattrs = i;
773         }
774
775         /* Get the first attribute to index */
776         if (bi->bi_linear_index && !index_nattrs) {
777                 index_nattrs = bi->bi_nattrs - 1;
778                 bi->bi_nattrs = 1;
779         }
780
781         e = bdb_tool_entry_get( be, id );
782
783         if( e == NULL ) {
784                 Debug( LDAP_DEBUG_ANY,
785                         LDAP_XSTRING(bdb_tool_entry_reindex)
786                         ": could not locate id=%ld\n",
787                         (long) id, 0, 0 );
788                 return -1;
789         }
790
791         if (! (slapMode & SLAP_TOOL_QUICK)) {
792         rc = TXN_BEGIN( bi->bi_dbenv, NULL, &tid, bi->bi_db_opflags );
793         if( rc != 0 ) {
794                 Debug( LDAP_DEBUG_ANY,
795                         "=> " LDAP_XSTRING(bdb_tool_entry_reindex) ": "
796                         "txn_begin failed: %s (%d)\n",
797                         db_strerror(rc), rc, 0 );
798                 goto done;
799         }
800         }
801         
802         /*
803          * just (re)add them for now
804          * assume that some other routine (not yet implemented)
805          * will zap index databases
806          *
807          */
808
809         Debug( LDAP_DEBUG_TRACE,
810                 "=> " LDAP_XSTRING(bdb_tool_entry_reindex) "( %ld, \"%s\" )\n",
811                 (long) id, e->e_dn, 0 );
812
813         op.o_hdr = &ohdr;
814         op.o_bd = be;
815         op.o_tmpmemctx = NULL;
816         op.o_tmpmfuncs = &ch_mfuncs;
817
818         rc = bdb_tool_index_add( &op, tid, e );
819
820 done:
821         if( rc == 0 ) {
822                 if (! (slapMode & SLAP_TOOL_QUICK)) {
823                 rc = TXN_COMMIT( tid, 0 );
824                 if( rc != 0 ) {
825                         Debug( LDAP_DEBUG_ANY,
826                                 "=> " LDAP_XSTRING(bdb_tool_entry_reindex)
827                                 ": txn_commit failed: %s (%d)\n",
828                                 db_strerror(rc), rc, 0 );
829                         e->e_id = NOID;
830                 }
831                 }
832
833         } else {
834                 if (! (slapMode & SLAP_TOOL_QUICK)) {
835                 TXN_ABORT( tid );
836                 Debug( LDAP_DEBUG_ANY,
837                         "=> " LDAP_XSTRING(bdb_tool_entry_reindex)
838                         ": txn_aborted! %s (%d)\n",
839                         db_strerror(rc), rc, 0 );
840                 }
841                 e->e_id = NOID;
842         }
843         bdb_entry_release( &op, e, 0 );
844
845         return rc;
846 }
847
848 ID bdb_tool_entry_modify(
849         BackendDB *be,
850         Entry *e,
851         struct berval *text )
852 {
853         int rc;
854         struct bdb_info *bdb = (struct bdb_info *) be->be_private;
855         DB_TXN *tid = NULL;
856         Operation op = {0};
857         Opheader ohdr = {0};
858
859         assert( be != NULL );
860         assert( slapMode & SLAP_TOOL_MODE );
861
862         assert( text != NULL );
863         assert( text->bv_val != NULL );
864         assert( text->bv_val[0] == '\0' );      /* overconservative? */
865
866         assert ( e->e_id != NOID );
867
868         Debug( LDAP_DEBUG_TRACE,
869                 "=> " LDAP_XSTRING(bdb_tool_entry_modify) "( %ld, \"%s\" )\n",
870                 (long) e->e_id, e->e_dn, 0 );
871
872         if (! (slapMode & SLAP_TOOL_QUICK)) {
873                 if( cursor ) {
874                         cursor->c_close( cursor );
875                         cursor = NULL;
876                 }
877                 rc = TXN_BEGIN( bdb->bi_dbenv, NULL, &tid, 
878                         bdb->bi_db_opflags );
879                 if( rc != 0 ) {
880                         snprintf( text->bv_val, text->bv_len,
881                                 "txn_begin failed: %s (%d)",
882                                 db_strerror(rc), rc );
883                         Debug( LDAP_DEBUG_ANY,
884                                 "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n",
885                                  text->bv_val, 0, 0 );
886                         return NOID;
887                 }
888         }
889
890         op.o_hdr = &ohdr;
891         op.o_bd = be;
892         op.o_tmpmemctx = NULL;
893         op.o_tmpmfuncs = &ch_mfuncs;
894
895         /* id2entry index */
896         rc = bdb_id2entry_update( be, tid, e );
897         if( rc != 0 ) {
898                 snprintf( text->bv_val, text->bv_len,
899                                 "id2entry_add failed: %s (%d)",
900                                 db_strerror(rc), rc );
901                 Debug( LDAP_DEBUG_ANY,
902                         "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n",
903                         text->bv_val, 0, 0 );
904                 goto done;
905         }
906
907 done:
908         if( rc == 0 ) {
909                 if (! (slapMode & SLAP_TOOL_QUICK)) {
910                 rc = TXN_COMMIT( tid, 0 );
911                 if( rc != 0 ) {
912                         snprintf( text->bv_val, text->bv_len,
913                                         "txn_commit failed: %s (%d)",
914                                         db_strerror(rc), rc );
915                         Debug( LDAP_DEBUG_ANY,
916                                 "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": "
917                                 "%s\n", text->bv_val, 0, 0 );
918                         e->e_id = NOID;
919                 }
920                 }
921
922         } else {
923                 if (! (slapMode & SLAP_TOOL_QUICK)) {
924                 TXN_ABORT( tid );
925                 snprintf( text->bv_val, text->bv_len,
926                         "txn_aborted! %s (%d)",
927                         db_strerror(rc), rc );
928                 Debug( LDAP_DEBUG_ANY,
929                         "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n",
930                         text->bv_val, 0, 0 );
931                 }
932                 e->e_id = NOID;
933         }
934
935         return e->e_id;
936 }
937
938 #ifdef BDB_TOOL_IDL_CACHING
939 static int
940 bdb_tool_idl_cmp( const void *v1, const void *v2 )
941 {
942         const bdb_tool_idl_cache *c1 = v1, *c2 = v2;
943         int rc;
944
945         if (( rc = c1->kstr.bv_len - c2->kstr.bv_len )) return rc;
946         return memcmp( c1->kstr.bv_val, c2->kstr.bv_val, c1->kstr.bv_len );
947 }
948
949 static int
950 bdb_tool_idl_flush_one( void *v1, void *arg )
951 {
952         bdb_tool_idl_cache *ic = v1;
953         DB *db = arg;
954         struct bdb_info *bdb = bdb_tool_info;
955         bdb_tool_idl_cache_entry *ice;
956         DBC *curs;
957         DBT key, data;
958         int i, rc;
959         ID id, nid;
960
961         /* Freshly allocated, ignore it */
962         if ( !ic->head && ic->count <= BDB_IDL_DB_SIZE ) {
963                 return 0;
964         }
965
966         rc = db->cursor( db, NULL, &curs, 0 );
967         if ( rc )
968                 return -1;
969
970         DBTzero( &key );
971         DBTzero( &data );
972
973         bv2DBT( &ic->kstr, &key );
974
975         data.size = data.ulen = sizeof( ID );
976         data.flags = DB_DBT_USERMEM;
977         data.data = &nid;
978
979         rc = curs->c_get( curs, &key, &data, DB_SET );
980         /* If key already exists and we're writing a range... */
981         if ( rc == 0 && ic->count > BDB_IDL_DB_SIZE ) {
982                 /* If it's not currently a range, must delete old info */
983                 if ( nid ) {
984                         /* Skip lo */
985                         while ( curs->c_get( curs, &key, &data, DB_NEXT_DUP ) == 0 )
986                                 curs->c_del( curs, 0 );
987
988                         nid = 0;
989                         /* Store range marker */
990                         curs->c_put( curs, &key, &data, DB_KEYFIRST );
991                 } else {
992                         
993                         /* Skip lo */
994                         rc = curs->c_get( curs, &key, &data, DB_NEXT_DUP );
995
996                         /* Get hi */
997                         rc = curs->c_get( curs, &key, &data, DB_NEXT_DUP );
998
999                         /* Delete hi */
1000                         curs->c_del( curs, 0 );
1001                 }
1002                 BDB_ID2DISK( ic->last, &nid );
1003                 curs->c_put( curs, &key, &data, DB_KEYLAST );
1004                 rc = 0;
1005         } else if ( rc && rc != DB_NOTFOUND ) {
1006                 rc = -1;
1007         } else if ( ic->count > BDB_IDL_DB_SIZE ) {
1008                 /* range, didn't exist before */
1009                 nid = 0;
1010                 rc = curs->c_put( curs, &key, &data, DB_KEYLAST );
1011                 if ( rc == 0 ) {
1012                         BDB_ID2DISK( ic->first, &nid );
1013                         rc = curs->c_put( curs, &key, &data, DB_KEYLAST );
1014                         if ( rc == 0 ) {
1015                                 BDB_ID2DISK( ic->last, &nid );
1016                                 rc = curs->c_put( curs, &key, &data, DB_KEYLAST );
1017                         }
1018                 }
1019                 if ( rc ) {
1020                         rc = -1;
1021                 }
1022         } else {
1023                 int n;
1024
1025                 /* Just a normal write */
1026                 rc = 0;
1027                 for ( ice = ic->head, n=0; ice; ice = ice->next, n++ ) {
1028                         int end;
1029                         if ( ice->next ) {
1030                                 end = IDBLOCK;
1031                         } else {
1032                                 end = ic->count & (IDBLOCK-1);
1033                                 if ( !end )
1034                                         end = IDBLOCK;
1035                         }
1036                         for ( i=0; i<end; i++ ) {
1037                                 if ( !ice->ids[i] ) continue;
1038                                 BDB_ID2DISK( ice->ids[i], &nid );
1039                                 rc = curs->c_put( curs, &key, &data, DB_NODUPDATA );
1040                                 if ( rc ) {
1041                                         if ( rc == DB_KEYEXIST ) {
1042                                                 rc = 0;
1043                                                 continue;
1044                                         }
1045                                         rc = -1;
1046                                         break;
1047                                 }
1048                         }
1049                         if ( rc ) {
1050                                 rc = -1;
1051                                 break;
1052                         }
1053                 }
1054                 if ( ic->head ) {
1055                         ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
1056                         ic->tail->next = bdb_tool_idl_free_list;
1057                         bdb_tool_idl_free_list = ic->head;
1058                         bdb->bi_idl_cache_size -= n;
1059                         ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
1060                 }
1061         }
1062         if ( ic != db->app_private ) {
1063                 ch_free( ic );
1064         } else {
1065                 ic->head = ic->tail = NULL;
1066         }
1067         curs->c_close( curs );
1068         return rc;
1069 }
1070
1071 static int
1072 bdb_tool_idl_flush_db( DB *db, bdb_tool_idl_cache *ic )
1073 {
1074         Avlnode *root = db->app_private;
1075         int rc;
1076
1077         db->app_private = ic;
1078         rc = avl_apply( root, bdb_tool_idl_flush_one, db, -1, AVL_INORDER );
1079         avl_free( root, NULL );
1080         db->app_private = NULL;
1081         if ( rc != -1 )
1082                 rc = 0;
1083         return rc;
1084 }
1085
1086 static int
1087 bdb_tool_idl_flush( BackendDB *be )
1088 {
1089         struct bdb_info *bdb = (struct bdb_info *) be->be_private;
1090         DB *db;
1091         Avlnode *root;
1092         int i, rc = 0;
1093
1094         for ( i=BDB_NDB; i < bdb->bi_ndatabases; i++ ) {
1095                 db = bdb->bi_databases[i]->bdi_db;
1096                 if ( !db->app_private ) continue;
1097                 rc = bdb_tool_idl_flush_db( db, NULL );
1098                 if ( rc )
1099                         break;
1100         }
1101         if ( !rc ) {
1102                 bdb->bi_idl_cache_size = 0;
1103         }
1104         return rc;
1105 }
1106
1107 int bdb_tool_idl_add(
1108         BackendDB *be,
1109         DB *db,
1110         DB_TXN *txn,
1111         DBT *key,
1112         ID id )
1113 {
1114         struct bdb_info *bdb = (struct bdb_info *) be->be_private;
1115         bdb_tool_idl_cache *ic, itmp;
1116         bdb_tool_idl_cache_entry *ice;
1117         int rc;
1118
1119         if ( !bdb->bi_idl_cache_max_size )
1120                 return bdb_idl_insert_key( be, db, txn, key, id );
1121
1122         DBT2bv( key, &itmp.kstr );
1123
1124         ic = avl_find( (Avlnode *)db->app_private, &itmp, bdb_tool_idl_cmp );
1125
1126         /* No entry yet, create one */
1127         if ( !ic ) {
1128                 DBC *curs;
1129                 DBT data;
1130                 ID nid;
1131                 int rc;
1132
1133                 ic = ch_malloc( sizeof( bdb_tool_idl_cache ) + itmp.kstr.bv_len );
1134                 ic->kstr.bv_len = itmp.kstr.bv_len;
1135                 ic->kstr.bv_val = (char *)(ic+1);
1136                 AC_MEMCPY( ic->kstr.bv_val, itmp.kstr.bv_val, ic->kstr.bv_len );
1137                 ic->head = ic->tail = NULL;
1138                 ic->last = 0;
1139                 ic->count = 0;
1140                 avl_insert( (Avlnode **)&db->app_private, ic, bdb_tool_idl_cmp,
1141                         avl_dup_error );
1142
1143                 /* load existing key count here */
1144                 rc = db->cursor( db, NULL, &curs, 0 );
1145                 if ( rc ) return rc;
1146
1147                 data.ulen = sizeof( ID );
1148                 data.flags = DB_DBT_USERMEM;
1149                 data.data = &nid;
1150                 rc = curs->c_get( curs, key, &data, DB_SET );
1151                 if ( rc == 0 ) {
1152                         if ( nid == 0 ) {
1153                                 ic->count = BDB_IDL_DB_SIZE+1;
1154                         } else {
1155                                 db_recno_t count;
1156
1157                                 curs->c_count( curs, &count, 0 );
1158                                 ic->count = count;
1159                                 BDB_DISK2ID( &nid, &ic->first );
1160                         }
1161                 }
1162                 curs->c_close( curs );
1163         }
1164         /* are we a range already? */
1165         if ( ic->count > BDB_IDL_DB_SIZE ) {
1166                 ic->last = id;
1167                 return 0;
1168         /* Are we at the limit, and converting to a range? */
1169         } else if ( ic->count == BDB_IDL_DB_SIZE ) {
1170                 int n;
1171                 for ( ice = ic->head, n=0; ice; ice = ice->next, n++ )
1172                         /* counting */ ;
1173                 if ( n ) {
1174                         ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
1175                         ic->tail->next = bdb_tool_idl_free_list;
1176                         bdb_tool_idl_free_list = ic->head;
1177                         bdb->bi_idl_cache_size -= n;
1178                         ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
1179                 }
1180                 ic->head = ic->tail = NULL;
1181                 ic->last = id;
1182                 ic->count++;
1183                 return 0;
1184         }
1185         /* No free block, create that too */
1186         if ( !ic->tail || ( ic->count & (IDBLOCK-1)) == 0) {
1187                 ice = NULL;
1188                 ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
1189                 if ( bdb->bi_idl_cache_size >= bdb->bi_idl_cache_max_size ) {
1190                         ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
1191                         rc = bdb_tool_idl_flush_db( db, ic );
1192                         if ( rc )
1193                                 return rc;
1194                         avl_insert( (Avlnode **)&db->app_private, ic, bdb_tool_idl_cmp,
1195                                 avl_dup_error );
1196                         ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
1197                 }
1198                 bdb->bi_idl_cache_size++;
1199                 if ( bdb_tool_idl_free_list ) {
1200                         ice = bdb_tool_idl_free_list;
1201                         bdb_tool_idl_free_list = ice->next;
1202                 }
1203                 ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
1204                 if ( !ice ) {
1205                         ice = ch_malloc( sizeof( bdb_tool_idl_cache_entry ));
1206                 }
1207                 memset( ice, 0, sizeof( *ice ));
1208                 if ( !ic->head ) {
1209                         ic->head = ice;
1210                 } else {
1211                         ic->tail->next = ice;
1212                 }
1213                 ic->tail = ice;
1214                 if ( !ic->count )
1215                         ic->first = id;
1216         }
1217         ice = ic->tail;
1218         ice->ids[ ic->count & (IDBLOCK-1) ] = id;
1219         ic->count++;
1220
1221         return 0;
1222 }
1223 #endif
1224
1225 #ifdef USE_TRICKLE
1226 static void *
1227 bdb_tool_trickle_task( void *ctx, void *ptr )
1228 {
1229         DB_ENV *env = ptr;
1230         int wrote;
1231
1232         ldap_pvt_thread_mutex_lock( &bdb_tool_trickle_mutex );
1233         while ( 1 ) {
1234                 ldap_pvt_thread_cond_wait( &bdb_tool_trickle_cond,
1235                         &bdb_tool_trickle_mutex );
1236                 if ( slapd_shutdown )
1237                         break;
1238                 env->memp_trickle( env, 30, &wrote );
1239         }
1240         ldap_pvt_thread_mutex_unlock( &bdb_tool_trickle_mutex );
1241
1242         return NULL;
1243 }
1244 #endif
1245
1246 static void *
1247 bdb_tool_index_task( void *ctx, void *ptr )
1248 {
1249         int base = *(int *)ptr;
1250
1251         free( ptr );
1252         while ( 1 ) {
1253                 ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex );
1254                 bdb_tool_index_tcount--;
1255                 if ( !bdb_tool_index_tcount )
1256                         ldap_pvt_thread_cond_signal( &bdb_tool_index_cond_main );
1257                 ldap_pvt_thread_cond_wait( &bdb_tool_index_cond_work,
1258                         &bdb_tool_index_mutex );
1259                 ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex );
1260                 if ( slapd_shutdown )
1261                         break;
1262
1263                 bdb_tool_index_threads[base] = bdb_index_recrun( bdb_tool_ix_op,
1264                         bdb_tool_info, bdb_tool_index_rec, bdb_tool_ix_id, base );
1265         }
1266
1267         return NULL;
1268 }