]> git.sur5r.net Git - openldap/blob - servers/slapd/back-mdb/tools.c
78e1e36b98816d66aa308732ca087ae7c51b8d8e
[openldap] / servers / slapd / back-mdb / tools.c
1 /* tools.c - tools for slap tools */
2 /* $OpenLDAP$ */
3 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
4  *
5  * Copyright 2011-2012 The OpenLDAP Foundation.
6  * All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted only as authorized by the OpenLDAP
10  * Public License.
11  *
12  * A copy of this license is available in the file LICENSE in the
13  * top-level directory of the distribution or, alternatively, at
14  * <http://www.OpenLDAP.org/license.html>.
15  */
16
17 #include "portable.h"
18
19 #include <stdio.h>
20 #include <ac/string.h>
21 #include <ac/errno.h>
22
23 #define AVL_INTERNAL
24 #include "back-mdb.h"
25 #include "idl.h"
26
27 #ifdef MDB_TOOL_IDL_CACHING
28 static int mdb_tool_idl_flush( BackendDB *be, MDB_txn *txn );
29
30 #define IDBLOCK 1024
31
32 typedef struct mdb_tool_idl_cache_entry {
33         struct mdb_tool_idl_cache_entry *next;
34         ID ids[IDBLOCK];
35 } mdb_tool_idl_cache_entry;
36
37 typedef struct mdb_tool_idl_cache {
38         struct berval kstr;
39         mdb_tool_idl_cache_entry *head, *tail;
40         ID first, last;
41         int count;
42         short offset;
43         short flags;
44 } mdb_tool_idl_cache;
45 #define WAS_FOUND       0x01
46 #define WAS_RANGE       0x02
47
48 #define MDB_TOOL_IDL_FLUSH(be, txn)     mdb_tool_idl_flush(be, txn)
49 #else
50 #define MDB_TOOL_IDL_FLUSH(be, txn)
51 #endif /* MDB_TOOL_IDL_CACHING */
52
53 static MDB_txn *txn = NULL, *txi = NULL;
54 static MDB_cursor *cursor = NULL, *idcursor = NULL;
55 static MDB_cursor *mcp = NULL, *mcd = NULL;
56 static MDB_val key, data;
57 static ID previd = NOID;
58
59 typedef struct dn_id {
60         ID id;
61         struct berval dn;
62 } dn_id;
63
64 #define HOLE_SIZE       4096
65 static dn_id hbuf[HOLE_SIZE], *holes = hbuf;
66 static unsigned nhmax = HOLE_SIZE;
67 static unsigned nholes;
68
69 static struct berval    *tool_base;
70 static int              tool_scope;
71 static Filter           *tool_filter;
72 static Entry            *tool_next_entry;
73
74 static ID mdb_tool_ix_id;
75 static Operation *mdb_tool_ix_op;
76 static MDB_txn *mdb_tool_ix_txn;
77 static int mdb_tool_index_tcount, mdb_tool_threads;
78 static IndexRec *mdb_tool_index_rec;
79 static struct mdb_info *mdb_tool_info;
80 static ldap_pvt_thread_mutex_t mdb_tool_index_mutex;
81 static ldap_pvt_thread_cond_t mdb_tool_index_cond_main;
82 static ldap_pvt_thread_cond_t mdb_tool_index_cond_work;
83 static void * mdb_tool_index_task( void *ctx, void *ptr );
84
85 static int      mdb_writes, mdb_writes_per_commit;
86
87 static int
88 mdb_tool_entry_get_int( BackendDB *be, ID id, Entry **ep );
89
90 int mdb_tool_entry_open(
91         BackendDB *be, int mode )
92 {
93         /* In Quick mode, commit once per 1000 entries */
94         mdb_writes = 0;
95         if ( slapMode & SLAP_TOOL_QUICK )
96                 mdb_writes_per_commit = 1000;
97         else
98                 mdb_writes_per_commit = 1;
99
100         /* Set up for threaded slapindex */
101         if (( slapMode & (SLAP_TOOL_QUICK|SLAP_TOOL_READONLY)) == SLAP_TOOL_QUICK ) {
102                 if ( !mdb_tool_info ) {
103                         struct mdb_info *mdb = (struct mdb_info *) be->be_private;
104                         ldap_pvt_thread_mutex_init( &mdb_tool_index_mutex );
105                         ldap_pvt_thread_cond_init( &mdb_tool_index_cond_main );
106                         ldap_pvt_thread_cond_init( &mdb_tool_index_cond_work );
107                         if ( mdb->mi_nattrs ) {
108                                 int i;
109                                 mdb_tool_threads = slap_tool_thread_max - 1;
110                                 if ( mdb_tool_threads > 1 ) {
111                                         mdb_tool_index_rec = ch_calloc( mdb->mi_nattrs, sizeof( IndexRec ));
112                                         mdb_tool_index_tcount = mdb_tool_threads - 1;
113                                         for (i=1; i<mdb_tool_threads; i++) {
114                                                 int *ptr = ch_malloc( sizeof( int ));
115                                                 *ptr = i;
116                                                 ldap_pvt_thread_pool_submit( &connection_pool,
117                                                         mdb_tool_index_task, ptr );
118                                         }
119                                         mdb_tool_info = mdb;
120                                 }
121                         }
122                 }
123         }
124
125         return 0;
126 }
127
128 int mdb_tool_entry_close(
129         BackendDB *be )
130 {
131         if ( mdb_tool_info ) {
132                 slapd_shutdown = 1;
133                 ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex );
134
135                 /* There might still be some threads starting */
136                 while ( mdb_tool_index_tcount > 0 ) {
137                         ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main,
138                                         &mdb_tool_index_mutex );
139                 }
140
141                 mdb_tool_index_tcount = mdb_tool_threads - 1;
142                 ldap_pvt_thread_cond_broadcast( &mdb_tool_index_cond_work );
143
144                 /* Make sure all threads are stopped */
145                 while ( mdb_tool_index_tcount > 0 ) {
146                         ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main,
147                                 &mdb_tool_index_mutex );
148                 }
149                 ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex );
150
151                 mdb_tool_info = NULL;
152                 slapd_shutdown = 0;
153                 ch_free( mdb_tool_index_rec );
154                 mdb_tool_index_tcount = mdb_tool_threads - 1;
155         }
156
157         if( idcursor ) {
158                 mdb_cursor_close( idcursor );
159                 idcursor = NULL;
160         }
161         if( cursor ) {
162                 mdb_cursor_close( cursor );
163                 cursor = NULL;
164         }
165         if( txn ) {
166                 MDB_TOOL_IDL_FLUSH( be, txn );
167                 if ( mdb_txn_commit( txn ))
168                         return -1;
169                 txn = NULL;
170         }
171
172         if( nholes ) {
173                 unsigned i;
174                 fprintf( stderr, "Error, entries missing!\n");
175                 for (i=0; i<nholes; i++) {
176                         fprintf(stderr, "  entry %ld: %s\n",
177                                 holes[i].id, holes[i].dn.bv_val);
178                 }
179                 nholes = 0;
180                 return -1;
181         }
182
183         return 0;
184 }
185
186 ID
187 mdb_tool_entry_first_x(
188         BackendDB *be,
189         struct berval *base,
190         int scope,
191         Filter *f )
192 {
193         tool_base = base;
194         tool_scope = scope;
195         tool_filter = f;
196
197         return mdb_tool_entry_next( be );
198 }
199
200 ID mdb_tool_entry_next(
201         BackendDB *be )
202 {
203         int rc;
204         ID id;
205         struct mdb_info *mdb;
206
207         assert( be != NULL );
208         assert( slapMode & SLAP_TOOL_MODE );
209
210         mdb = (struct mdb_info *) be->be_private;
211         assert( mdb != NULL );
212
213         if ( !txn ) {
214                 rc = mdb_txn_begin( mdb->mi_dbenv, NULL, MDB_RDONLY, &txn );
215                 if ( rc )
216                         return NOID;
217                 rc = mdb_cursor_open( txn, mdb->mi_id2entry, &cursor );
218                 if ( rc ) {
219                         mdb_txn_abort( txn );
220                         return NOID;
221                 }
222         }
223
224 next:;
225         rc = mdb_cursor_get( cursor, &key, &data, MDB_NEXT );
226
227         if( rc ) {
228                 return NOID;
229         }
230
231         previd = *(ID *)key.mv_data;
232         id = previd;
233
234         if ( tool_filter || tool_base ) {
235                 static Operation op = {0};
236                 static Opheader ohdr = {0};
237
238                 op.o_hdr = &ohdr;
239                 op.o_bd = be;
240                 op.o_tmpmemctx = NULL;
241                 op.o_tmpmfuncs = &ch_mfuncs;
242
243                 if ( tool_next_entry ) {
244                         mdb_entry_release( &op, tool_next_entry, 0 );
245                         tool_next_entry = NULL;
246                 }
247
248                 rc = mdb_tool_entry_get_int( be, id, &tool_next_entry );
249                 if ( rc == LDAP_NO_SUCH_OBJECT ) {
250                         goto next;
251                 }
252
253                 assert( tool_next_entry != NULL );
254
255                 if ( tool_filter && test_filter( NULL, tool_next_entry, tool_filter ) != LDAP_COMPARE_TRUE )
256                 {
257                         mdb_entry_release( &op, tool_next_entry, 0 );
258                         tool_next_entry = NULL;
259                         goto next;
260                 }
261         }
262
263         return id;
264 }
265
266 ID mdb_tool_dn2id_get(
267         Backend *be,
268         struct berval *dn
269 )
270 {
271         struct mdb_info *mdb;
272         Operation op = {0};
273         Opheader ohdr = {0};
274         ID id;
275         int rc;
276
277         if ( BER_BVISEMPTY(dn) )
278                 return 0;
279
280         mdb = (struct mdb_info *) be->be_private;
281
282         if ( !txn ) {
283                 rc = mdb_txn_begin( mdb->mi_dbenv, NULL, (slapMode & SLAP_TOOL_READONLY) != 0 ?
284                         MDB_RDONLY : 0, &txn );
285                 if ( rc )
286                         return NOID;
287         }
288
289         op.o_hdr = &ohdr;
290         op.o_bd = be;
291         op.o_tmpmemctx = NULL;
292         op.o_tmpmfuncs = &ch_mfuncs;
293
294         rc = mdb_dn2id( &op, txn, NULL, dn, &id, NULL, NULL );
295         if ( rc == MDB_NOTFOUND )
296                 return NOID;
297
298         return id;
299 }
300
301 static int
302 mdb_tool_entry_get_int( BackendDB *be, ID id, Entry **ep )
303 {
304         Operation op = {0};
305         Opheader ohdr = {0};
306
307         Entry *e = NULL;
308         struct berval dn = BER_BVNULL, ndn = BER_BVNULL;
309         int rc;
310
311         assert( be != NULL );
312         assert( slapMode & SLAP_TOOL_MODE );
313
314         if ( ( tool_filter || tool_base ) && id == previd && tool_next_entry != NULL ) {
315                 *ep = tool_next_entry;
316                 tool_next_entry = NULL;
317                 return LDAP_SUCCESS;
318         }
319
320         if ( id != previd ) {
321                 key.mv_size = sizeof(ID);
322                 key.mv_data = &id;
323                 rc = mdb_cursor_get( cursor, &key, &data, MDB_SET );
324                 if ( rc ) {
325                         rc = LDAP_OTHER;
326                         goto done;
327                 }
328         }
329
330         op.o_hdr = &ohdr;
331         op.o_bd = be;
332         op.o_tmpmemctx = NULL;
333         op.o_tmpmfuncs = &ch_mfuncs;
334         if ( slapMode & SLAP_TOOL_READONLY ) {
335                 rc = mdb_id2name( &op, txn, &idcursor, id, &dn, &ndn );
336                 if ( rc  ) {
337                         rc = LDAP_OTHER;
338                         mdb_entry_return( &op, e );
339                         e = NULL;
340                         goto done;
341                 }
342                 if ( tool_base != NULL ) {
343                         if ( !dnIsSuffixScope( &ndn, tool_base, tool_scope ) ) {
344                                 ch_free( dn.bv_val );
345                                 ch_free( ndn.bv_val );
346                                 rc = LDAP_NO_SUCH_OBJECT;
347                         }
348                 }
349         }
350         rc = mdb_entry_decode( &op, &data, &e );
351         e->e_id = id;
352         if ( !BER_BVISNULL( &dn )) {
353                 e->e_name = dn;
354                 e->e_nname = ndn;
355         } else {
356                 e->e_name.bv_val = NULL;
357                 e->e_nname.bv_val = NULL;
358         }
359
360 done:
361         if ( e != NULL ) {
362                 *ep = e;
363         }
364
365         return rc;
366 }
367
368 Entry*
369 mdb_tool_entry_get( BackendDB *be, ID id )
370 {
371         Entry *e = NULL;
372
373         (void)mdb_tool_entry_get_int( be, id, &e );
374         return e;
375 }
376
377 static int mdb_tool_next_id(
378         Operation *op,
379         MDB_txn *tid,
380         Entry *e,
381         struct berval *text,
382         int hole )
383 {
384         struct mdb_info *mdb = (struct mdb_info *) op->o_bd->be_private;
385         struct berval dn = e->e_name;
386         struct berval ndn = e->e_nname;
387         struct berval pdn, npdn, nmatched;
388         ID id, pid = 0;
389         int rc;
390
391         if (ndn.bv_len == 0) {
392                 e->e_id = 0;
393                 return 0;
394         }
395
396         rc = mdb_dn2id( op, tid, mcp, &ndn, &id, NULL, &nmatched );
397         if ( rc == MDB_NOTFOUND ) {
398                 if ( !be_issuffix( op->o_bd, &ndn ) ) {
399                         ID eid = e->e_id;
400                         dnParent( &ndn, &npdn );
401                         if ( nmatched.bv_len != npdn.bv_len ) {
402                                 dnParent( &dn, &pdn );
403                                 e->e_name = pdn;
404                                 e->e_nname = npdn;
405                                 rc = mdb_tool_next_id( op, tid, e, text, 1 );
406                                 e->e_name = dn;
407                                 e->e_nname = ndn;
408                                 if ( rc ) {
409                                         return rc;
410                                 }
411                                 /* If parent didn't exist, it was created just now
412                                  * and its ID is now in e->e_id. Make sure the current
413                                  * entry gets added under the new parent ID.
414                                  */
415                                 if ( eid != e->e_id ) {
416                                         pid = e->e_id;
417                                 }
418                         } else {
419                                 pid = id;
420                         }
421                 }
422                 rc = mdb_next_id( op->o_bd, idcursor, &e->e_id );
423                 if ( rc ) {
424                         snprintf( text->bv_val, text->bv_len,
425                                 "next_id failed: %s (%d)",
426                                 mdb_strerror(rc), rc );
427                 Debug( LDAP_DEBUG_ANY,
428                         "=> mdb_tool_next_id: %s\n", text->bv_val, 0, 0 );
429                         return rc;
430                 }
431                 rc = mdb_dn2id_add( op, mcp, mcd, pid, e );
432                 if ( rc ) {
433                         snprintf( text->bv_val, text->bv_len,
434                                 "dn2id_add failed: %s (%d)",
435                                 mdb_strerror(rc), rc );
436                         Debug( LDAP_DEBUG_ANY,
437                                 "=> mdb_tool_next_id: %s\n", text->bv_val, 0, 0 );
438                 } else if ( hole ) {
439                         MDB_val key, data;
440                         if ( nholes == nhmax - 1 ) {
441                                 if ( holes == hbuf ) {
442                                         holes = ch_malloc( nhmax * sizeof(dn_id) * 2 );
443                                         AC_MEMCPY( holes, hbuf, sizeof(hbuf) );
444                                 } else {
445                                         holes = ch_realloc( holes, nhmax * sizeof(dn_id) * 2 );
446                                 }
447                                 nhmax *= 2;
448                         }
449                         ber_dupbv( &holes[nholes].dn, &ndn );
450                         holes[nholes++].id = e->e_id;
451                         key.mv_size = sizeof(ID);
452                         key.mv_data = &e->e_id;
453                         data.mv_size = 0;
454                         data.mv_data = NULL;
455                         rc = mdb_cursor_put( idcursor, &key, &data, MDB_NOOVERWRITE );
456                         if ( rc == MDB_KEYEXIST )
457                                 rc = 0;
458                         if ( rc ) {
459                                 snprintf( text->bv_val, text->bv_len,
460                                         "dummy id2entry add failed: %s (%d)",
461                                         mdb_strerror(rc), rc );
462                                 Debug( LDAP_DEBUG_ANY,
463                                         "=> mdb_tool_next_id: %s\n", text->bv_val, 0, 0 );
464                         }
465                 }
466         } else if ( !hole ) {
467                 unsigned i, j;
468
469                 e->e_id = id;
470
471                 for ( i=0; i<nholes; i++) {
472                         if ( holes[i].id == e->e_id ) {
473                                 free(holes[i].dn.bv_val);
474                                 for (j=i;j<nholes;j++) holes[j] = holes[j+1];
475                                 holes[j].id = 0;
476                                 nholes--;
477                                 break;
478                         } else if ( holes[i].id > e->e_id ) {
479                                 break;
480                         }
481                 }
482         }
483         return rc;
484 }
485
486 static int
487 mdb_tool_index_add(
488         Operation *op,
489         MDB_txn *txn,
490         Entry *e )
491 {
492         struct mdb_info *mdb = (struct mdb_info *) op->o_bd->be_private;
493
494         if ( !mdb->mi_nattrs )
495                 return 0;
496
497         if ( mdb_tool_threads > 1 ) {
498                 IndexRec *ir;
499                 int i, rc;
500                 Attribute *a;
501
502                 ir = mdb_tool_index_rec;
503                 for (i=0; i<mdb->mi_nattrs; i++)
504                         ir[i].ir_attrs = NULL;
505
506                 for ( a = e->e_attrs; a != NULL; a = a->a_next ) {
507                         rc = mdb_index_recset( mdb, a, a->a_desc->ad_type,
508                                 &a->a_desc->ad_tags, ir );
509                         if ( rc )
510                                 return rc;
511                 }
512                 for (i=0; i<mdb->mi_nattrs; i++) {
513                         if ( !ir[i].ir_ai )
514                                 break;
515                         rc = mdb_cursor_open( txn, ir[i].ir_ai->ai_dbi,
516                                  &ir[i].ir_ai->ai_cursor );
517                         if ( rc )
518                                 return rc;
519                 }
520                 mdb_tool_ix_id = e->e_id;
521                 mdb_tool_ix_op = op;
522                 mdb_tool_ix_txn = txn;
523                 ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex );
524                 /* Wait for all threads to be ready */
525                 while ( mdb_tool_index_tcount ) {
526                         ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main,
527                                 &mdb_tool_index_mutex );
528                 }
529
530                 for ( i=1; i<mdb_tool_threads; i++ )
531                         mdb_tool_index_rec[i].ir_i = LDAP_BUSY;
532                 mdb_tool_index_tcount = mdb_tool_threads - 1;
533                 ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex );
534                 ldap_pvt_thread_cond_broadcast( &mdb_tool_index_cond_work );
535
536                 rc = mdb_index_recrun( op, txn, mdb, ir, e->e_id, 0 );
537                 if ( rc )
538                         return rc;
539                 ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex );
540                 for ( i=1; i<mdb_tool_threads; i++ ) {
541                         if ( mdb_tool_index_rec[i].ir_i == LDAP_BUSY ) {
542                                 ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main,
543                                         &mdb_tool_index_mutex );
544                                 i--;
545                                 continue;
546                         }
547                         if ( mdb_tool_index_rec[i].ir_i ) {
548                                 rc = mdb_tool_index_rec[i].ir_i;
549                                 break;
550                         }
551                 }
552                 ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex );
553                 return rc;
554         } else
555         {
556                 return mdb_index_entry_add( op, txn, e );
557         }
558 }
559
560 ID mdb_tool_entry_put(
561         BackendDB *be,
562         Entry *e,
563         struct berval *text )
564 {
565         int rc;
566         struct mdb_info *mdb;
567         Operation op = {0};
568         Opheader ohdr = {0};
569
570         assert( be != NULL );
571         assert( slapMode & SLAP_TOOL_MODE );
572
573         assert( text != NULL );
574         assert( text->bv_val != NULL );
575         assert( text->bv_val[0] == '\0' );      /* overconservative? */
576
577         Debug( LDAP_DEBUG_TRACE, "=> " LDAP_XSTRING(mdb_tool_entry_put)
578                 "( %ld, \"%s\" )\n", (long) e->e_id, e->e_dn, 0 );
579
580         mdb = (struct mdb_info *) be->be_private;
581
582         if ( !txn ) {
583                 rc = mdb_txn_begin( mdb->mi_dbenv, NULL, 0, &txn );
584                 if( rc != 0 ) {
585                         snprintf( text->bv_val, text->bv_len,
586                                 "txn_begin failed: %s (%d)",
587                                 mdb_strerror(rc), rc );
588                         Debug( LDAP_DEBUG_ANY,
589                                 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
590                                  text->bv_val, 0, 0 );
591                         return NOID;
592                 }
593                 rc = mdb_cursor_open( txn, mdb->mi_id2entry, &idcursor );
594                 if( rc != 0 ) {
595                         snprintf( text->bv_val, text->bv_len,
596                                 "cursor_open failed: %s (%d)",
597                                 mdb_strerror(rc), rc );
598                         Debug( LDAP_DEBUG_ANY,
599                                 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
600                                  text->bv_val, 0, 0 );
601                         return NOID;
602                 }
603                 rc = mdb_cursor_open( txn, mdb->mi_dn2id, &mcp );
604                 if( rc != 0 ) {
605                         snprintf( text->bv_val, text->bv_len,
606                                 "cursor_open failed: %s (%d)",
607                                 mdb_strerror(rc), rc );
608                         Debug( LDAP_DEBUG_ANY,
609                                 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
610                                  text->bv_val, 0, 0 );
611                         return NOID;
612                 }
613                 rc = mdb_cursor_open( txn, mdb->mi_dn2id, &mcd );
614                 if( rc != 0 ) {
615                         snprintf( text->bv_val, text->bv_len,
616                                 "cursor_open failed: %s (%d)",
617                                 mdb_strerror(rc), rc );
618                         Debug( LDAP_DEBUG_ANY,
619                                 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
620                                  text->bv_val, 0, 0 );
621                         return NOID;
622                 }
623         }
624
625         op.o_hdr = &ohdr;
626         op.o_bd = be;
627         op.o_tmpmemctx = NULL;
628         op.o_tmpmfuncs = &ch_mfuncs;
629
630         /* add dn2id indices */
631         rc = mdb_tool_next_id( &op, txn, e, text, 0 );
632         if( rc != 0 ) {
633                 goto done;
634         }
635
636         rc = mdb_tool_index_add( &op, txn, e );
637         if( rc != 0 ) {
638                 snprintf( text->bv_val, text->bv_len,
639                                 "index_entry_add failed: err=%d", rc );
640                 Debug( LDAP_DEBUG_ANY,
641                         "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
642                         text->bv_val, 0, 0 );
643                 goto done;
644         }
645
646
647         /* id2entry index */
648         rc = mdb_id2entry_add( &op, txn, idcursor, e );
649         if( rc != 0 ) {
650                 snprintf( text->bv_val, text->bv_len,
651                                 "id2entry_add failed: err=%d", rc );
652                 Debug( LDAP_DEBUG_ANY,
653                         "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
654                         text->bv_val, 0, 0 );
655                 goto done;
656         }
657
658 done:
659         if( rc == 0 ) {
660                 mdb_writes++;
661                 if ( mdb_writes >= mdb_writes_per_commit ) {
662                         unsigned i;
663                         MDB_TOOL_IDL_FLUSH( be, txn );
664                         rc = mdb_txn_commit( txn );
665                         for ( i=0; i<mdb->mi_nattrs; i++ )
666                                 mdb->mi_attrs[i]->ai_cursor = NULL;
667                         mdb_writes = 0;
668                         txn = NULL;
669                         idcursor = NULL;
670                         if( rc != 0 ) {
671                                 snprintf( text->bv_val, text->bv_len,
672                                                 "txn_commit failed: %s (%d)",
673                                                 mdb_strerror(rc), rc );
674                                 Debug( LDAP_DEBUG_ANY,
675                                         "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
676                                         text->bv_val, 0, 0 );
677                                 e->e_id = NOID;
678                         }
679                 }
680
681         } else {
682                 mdb_txn_abort( txn );
683                 txn = NULL;
684                 cursor = NULL;
685                 snprintf( text->bv_val, text->bv_len,
686                         "txn_aborted! %s (%d)",
687                         rc == LDAP_OTHER ? "Internal error" :
688                         mdb_strerror(rc), rc );
689                 Debug( LDAP_DEBUG_ANY,
690                         "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
691                         text->bv_val, 0, 0 );
692                 e->e_id = NOID;
693         }
694
695         return e->e_id;
696 }
697
698 int mdb_tool_entry_reindex(
699         BackendDB *be,
700         ID id,
701         AttributeDescription **adv )
702 {
703         struct mdb_info *mi = (struct mdb_info *) be->be_private;
704         int rc;
705         Entry *e;
706         Operation op = {0};
707         Opheader ohdr = {0};
708
709         Debug( LDAP_DEBUG_ARGS,
710                 "=> " LDAP_XSTRING(mdb_tool_entry_reindex) "( %ld )\n",
711                 (long) id, 0, 0 );
712         assert( tool_base == NULL );
713         assert( tool_filter == NULL );
714
715         /* No indexes configured, nothing to do. Could return an
716          * error here to shortcut things.
717          */
718         if (!mi->mi_attrs) {
719                 return 0;
720         }
721
722         /* Check for explicit list of attrs to index */
723         if ( adv ) {
724                 int i, j, n;
725
726                 if ( mi->mi_attrs[0]->ai_desc != adv[0] ) {
727                         /* count */
728                         for ( n = 0; adv[n]; n++ ) ;
729
730                         /* insertion sort */
731                         for ( i = 0; i < n; i++ ) {
732                                 AttributeDescription *ad = adv[i];
733                                 for ( j = i-1; j>=0; j--) {
734                                         if ( SLAP_PTRCMP( adv[j], ad ) <= 0 ) break;
735                                         adv[j+1] = adv[j];
736                                 }
737                                 adv[j+1] = ad;
738                         }
739                 }
740
741                 for ( i = 0; adv[i]; i++ ) {
742                         if ( mi->mi_attrs[i]->ai_desc != adv[i] ) {
743                                 for ( j = i+1; j < mi->mi_nattrs; j++ ) {
744                                         if ( mi->mi_attrs[j]->ai_desc == adv[i] ) {
745                                                 AttrInfo *ai = mi->mi_attrs[i];
746                                                 mi->mi_attrs[i] = mi->mi_attrs[j];
747                                                 mi->mi_attrs[j] = ai;
748                                                 break;
749                                         }
750                                 }
751                                 if ( j == mi->mi_nattrs ) {
752                                         Debug( LDAP_DEBUG_ANY,
753                                                 LDAP_XSTRING(mdb_tool_entry_reindex)
754                                                 ": no index configured for %s\n",
755                                                 adv[i]->ad_cname.bv_val, 0, 0 );
756                                         return -1;
757                                 }
758                         }
759                 }
760                 mi->mi_nattrs = i;
761         }
762
763         if ( slapMode & SLAP_TRUNCATE_MODE ) {
764                 int i;
765                 for ( i=0; i < mi->mi_nattrs; i++ ) {
766                         rc = mdb_drop( txn, mi->mi_attrs[i]->ai_dbi, 0 );
767                         if ( rc ) {
768                                 Debug( LDAP_DEBUG_ANY,
769                                         LDAP_XSTRING(mdb_tool_entry_reindex)
770                                         ": (Truncate) mdb_drop(%s) failed: %s (%d)\n",
771                                         mi->mi_attrs[i]->ai_desc->ad_type->sat_cname.bv_val,
772                                         mdb_strerror(rc), rc );
773                                 return -1;
774                         }
775                 }
776                 slapMode ^= SLAP_TRUNCATE_MODE;
777         }
778
779         e = mdb_tool_entry_get( be, id );
780
781         if( e == NULL ) {
782                 Debug( LDAP_DEBUG_ANY,
783                         LDAP_XSTRING(mdb_tool_entry_reindex)
784                         ": could not locate id=%ld\n",
785                         (long) id, 0, 0 );
786                 return -1;
787         }
788
789         if ( !txi ) {
790                 rc = mdb_txn_begin( mi->mi_dbenv, NULL, 0, &txi );
791                 if( rc != 0 ) {
792                         Debug( LDAP_DEBUG_ANY,
793                                 "=> " LDAP_XSTRING(mdb_tool_entry_reindex) ": "
794                                 "txn_begin failed: %s (%d)\n",
795                                 mdb_strerror(rc), rc, 0 );
796                         goto done;
797                 }
798         }
799
800         /*
801          * just (re)add them for now
802          * assume that some other routine (not yet implemented)
803          * will zap index databases
804          *
805          */
806
807         Debug( LDAP_DEBUG_TRACE,
808                 "=> " LDAP_XSTRING(mdb_tool_entry_reindex) "( %ld )\n",
809                 (long) id, 0, 0 );
810
811         op.o_hdr = &ohdr;
812         op.o_bd = be;
813         op.o_tmpmemctx = NULL;
814         op.o_tmpmfuncs = &ch_mfuncs;
815
816         rc = mdb_tool_index_add( &op, txi, e );
817
818 done:
819         if( rc == 0 ) {
820                 mdb_writes++;
821                 if ( mdb_writes >= mdb_writes_per_commit ) {
822                         unsigned i;
823                         MDB_TOOL_IDL_FLUSH( be, txi );
824                         rc = mdb_txn_commit( txi );
825                         for ( i=0; i<mi->mi_nattrs; i++ )
826                                 mi->mi_attrs[i]->ai_cursor = NULL;
827                         if( rc != 0 ) {
828                                 Debug( LDAP_DEBUG_ANY,
829                                         "=> " LDAP_XSTRING(mdb_tool_entry_reindex)
830                                         ": txn_commit failed: %s (%d)\n",
831                                         mdb_strerror(rc), rc, 0 );
832                                 e->e_id = NOID;
833                         }
834                         txi = NULL;
835                 }
836
837         } else {
838                 mdb_txn_abort( txi );
839                 Debug( LDAP_DEBUG_ANY,
840                         "=> " LDAP_XSTRING(mdb_tool_entry_reindex)
841                         ": txn_aborted! err=%d\n",
842                         rc, 0, 0 );
843                 e->e_id = NOID;
844                 txi = NULL;
845         }
846         mdb_entry_release( &op, e, 0 );
847
848         return rc;
849 }
850
851 ID mdb_tool_entry_modify(
852         BackendDB *be,
853         Entry *e,
854         struct berval *text )
855 {
856         int rc;
857         struct mdb_info *mdb;
858         MDB_txn *tid;
859         Operation op = {0};
860         Opheader ohdr = {0};
861
862         assert( be != NULL );
863         assert( slapMode & SLAP_TOOL_MODE );
864
865         assert( text != NULL );
866         assert( text->bv_val != NULL );
867         assert( text->bv_val[0] == '\0' );      /* overconservative? */
868
869         assert ( e->e_id != NOID );
870
871         Debug( LDAP_DEBUG_TRACE,
872                 "=> " LDAP_XSTRING(mdb_tool_entry_modify) "( %ld, \"%s\" )\n",
873                 (long) e->e_id, e->e_dn, 0 );
874
875         mdb = (struct mdb_info *) be->be_private;
876
877         if( cursor ) {
878                 mdb_cursor_close( cursor );
879                 cursor = NULL;
880         }
881         rc = mdb_txn_begin( mdb->mi_dbenv, NULL, 0, &tid );
882         if( rc != 0 ) {
883                 snprintf( text->bv_val, text->bv_len,
884                         "txn_begin failed: %s (%d)",
885                         mdb_strerror(rc), rc );
886                 Debug( LDAP_DEBUG_ANY,
887                         "=> " LDAP_XSTRING(mdb_tool_entry_modify) ": %s\n",
888                          text->bv_val, 0, 0 );
889                 return NOID;
890         }
891
892         op.o_hdr = &ohdr;
893         op.o_bd = be;
894         op.o_tmpmemctx = NULL;
895         op.o_tmpmfuncs = &ch_mfuncs;
896
897         /* id2entry index */
898         rc = mdb_id2entry_update( &op, tid, NULL, e );
899         if( rc != 0 ) {
900                 snprintf( text->bv_val, text->bv_len,
901                                 "id2entry_update failed: err=%d", rc );
902                 Debug( LDAP_DEBUG_ANY,
903                         "=> " LDAP_XSTRING(mdb_tool_entry_modify) ": %s\n",
904                         text->bv_val, 0, 0 );
905                 goto done;
906         }
907
908 done:
909         if( rc == 0 ) {
910                 rc = mdb_txn_commit( tid );
911                 if( rc != 0 ) {
912                         snprintf( text->bv_val, text->bv_len,
913                                         "txn_commit failed: %s (%d)",
914                                         mdb_strerror(rc), rc );
915                         Debug( LDAP_DEBUG_ANY,
916                                 "=> " LDAP_XSTRING(mdb_tool_entry_modify) ": "
917                                 "%s\n", text->bv_val, 0, 0 );
918                         e->e_id = NOID;
919                 }
920
921         } else {
922                 mdb_txn_abort( tid );
923                 snprintf( text->bv_val, text->bv_len,
924                         "txn_aborted! %s (%d)",
925                         mdb_strerror(rc), rc );
926                 Debug( LDAP_DEBUG_ANY,
927                         "=> " LDAP_XSTRING(mdb_tool_entry_modify) ": %s\n",
928                         text->bv_val, 0, 0 );
929                 e->e_id = NOID;
930         }
931
932         return e->e_id;
933 }
934
935 static void *
936 mdb_tool_index_task( void *ctx, void *ptr )
937 {
938         int base = *(int *)ptr;
939
940         free( ptr );
941         while ( 1 ) {
942                 ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex );
943                 mdb_tool_index_tcount--;
944                 if ( !mdb_tool_index_tcount )
945                         ldap_pvt_thread_cond_signal( &mdb_tool_index_cond_main );
946                 ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_work,
947                         &mdb_tool_index_mutex );
948                 if ( slapd_shutdown ) {
949                         mdb_tool_index_tcount--;
950                         if ( !mdb_tool_index_tcount )
951                                 ldap_pvt_thread_cond_signal( &mdb_tool_index_cond_main );
952                         ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex );
953                         break;
954                 }
955                 ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex );
956                 mdb_tool_index_rec[base].ir_i = mdb_index_recrun( mdb_tool_ix_op,
957                         mdb_tool_ix_txn,
958                         mdb_tool_info, mdb_tool_index_rec, mdb_tool_ix_id, base );
959         }
960
961         return NULL;
962 }
963
964 #ifdef MDB_TOOL_IDL_CACHING
965 static int
966 mdb_tool_idl_cmp( const void *v1, const void *v2 )
967 {
968         const mdb_tool_idl_cache *c1 = v1, *c2 = v2;
969         int rc;
970
971         if (( rc = c1->kstr.bv_len - c2->kstr.bv_len )) return rc;
972         return memcmp( c1->kstr.bv_val, c2->kstr.bv_val, c1->kstr.bv_len );
973 }
974
975 static int
976 mdb_tool_idl_flush_one( MDB_cursor *mc, AttrInfo *ai, mdb_tool_idl_cache *ic )
977 {
978         mdb_tool_idl_cache_entry *ice;
979         MDB_val key, data[2];
980         int i, rc;
981         ID id, nid;
982
983         /* Freshly allocated, ignore it */
984         if ( !ic->head && ic->count <= MDB_IDL_DB_SIZE ) {
985                 return 0;
986         }
987
988         key.mv_data = ic->kstr.bv_val;
989         key.mv_size = ic->kstr.bv_len;
990
991         if ( ic->count > MDB_IDL_DB_SIZE ) {
992                 while ( ic->flags & WAS_FOUND ) {
993                         rc = mdb_cursor_get( mc, &key, data, MDB_SET );
994                         if ( rc ) {
995                                 /* FIXME: find out why this happens */
996                                 ic->flags = 0;
997                                 break;
998                         }
999                         if ( ic->flags & WAS_RANGE ) {
1000                                 /* Skip lo */
1001                                 rc = mdb_cursor_get( mc, &key, data, MDB_NEXT_DUP );
1002
1003                                 /* Get hi */
1004                                 rc = mdb_cursor_get( mc, &key, data, MDB_NEXT_DUP );
1005
1006                                 /* Store range hi */
1007                                 data[0].mv_data = &ic->last;
1008                                 rc = mdb_cursor_put( mc, &key, data, MDB_CURRENT );
1009                         } else {
1010                                 /* Delete old data, replace with range */
1011                                 ic->first = *(ID *)data[0].mv_data;
1012                                 mdb_cursor_del( mc, MDB_NODUPDATA );
1013                         }
1014                         break;
1015                 }
1016                 if ( !(ic->flags & WAS_RANGE)) {
1017                         /* range, didn't exist before */
1018                         nid = 0;
1019                         data[0].mv_size = sizeof(ID);
1020                         data[0].mv_data = &nid;
1021                         rc = mdb_cursor_put( mc, &key, data, 0 );
1022                         if ( rc == 0 ) {
1023                                 data[0].mv_data = &ic->first;
1024                                 rc = mdb_cursor_put( mc, &key, data, 0 );
1025                                 if ( rc == 0 ) {
1026                                         data[0].mv_data = &ic->last;
1027                                         rc = mdb_cursor_put( mc, &key, data, 0 );
1028                                 }
1029                         }
1030                         if ( rc ) {
1031                                 rc = -1;
1032                         }
1033                 }
1034         } else {
1035                 /* Normal write */
1036                 int n;
1037
1038                 data[0].mv_size = sizeof(ID);
1039                 rc = 0;
1040                 i = ic->offset;
1041                 for ( ice = ic->head, n=0; ice; ice = ice->next, n++ ) {
1042                         int end;
1043                         if ( ice->next ) {
1044                                 end = IDBLOCK;
1045                         } else {
1046                                 end = ic->count & (IDBLOCK-1);
1047                                 if ( !end )
1048                                         end = IDBLOCK;
1049                         }
1050                         data[1].mv_size = end - i;
1051                         data[0].mv_data = &ice->ids[i];
1052                         i = 0;
1053                         rc = mdb_cursor_put( mc, &key, data, MDB_NODUPDATA|MDB_APPEND|MDB_MULTIPLE );
1054                         if ( rc ) {
1055                                 if ( rc == MDB_KEYEXIST ) {
1056                                         rc = 0;
1057                                         continue;
1058                                 }
1059                                 rc = -1;
1060                                 break;
1061                         }
1062                 }
1063                 if ( ic->head ) {
1064                         ic->tail->next = ai->ai_flist;
1065                         ai->ai_flist = ic->head;
1066                 }
1067         }
1068         ic->head = ai->ai_clist;
1069         ai->ai_clist = ic;
1070         return rc;
1071 }
1072
1073 static int
1074 mdb_tool_idl_flush_db( MDB_txn *txn, AttrInfo *ai )
1075 {
1076         MDB_cursor *mc;
1077         Avlnode *root;
1078         int rc;
1079
1080         mdb_cursor_open( txn, ai->ai_dbi, &mc );
1081         root = tavl_end( ai->ai_root, TAVL_DIR_LEFT );
1082         do {
1083                 rc = mdb_tool_idl_flush_one( mc, ai, root->avl_data );
1084                 if ( rc != -1 )
1085                         rc = 0;
1086         } while ((root = tavl_next(root, TAVL_DIR_RIGHT)));
1087         mdb_cursor_close( mc );
1088
1089         return rc;
1090 }
1091
1092 static int
1093 mdb_tool_idl_flush( BackendDB *be, MDB_txn *txn )
1094 {
1095         struct mdb_info *mdb = (struct mdb_info *) be->be_private;
1096         int rc = 0;
1097         unsigned int i, dbi;
1098
1099         for ( i=0; i < mdb->mi_nattrs; i++ ) {
1100                 if ( !mdb->mi_attrs[i]->ai_root ) continue;
1101                 rc = mdb_tool_idl_flush_db( txn, mdb->mi_attrs[i] );
1102                 tavl_free(mdb->mi_attrs[i]->ai_root, NULL);
1103                 mdb->mi_attrs[i]->ai_root = NULL;
1104                 if ( rc )
1105                         break;
1106         }
1107         return rc;
1108 }
1109
1110 int mdb_tool_idl_add(
1111         MDB_cursor *mc,
1112         struct berval *keys,
1113         ID id )
1114 {
1115         MDB_dbi dbi;
1116         mdb_tool_idl_cache *ic, itmp;
1117         mdb_tool_idl_cache_entry *ice;
1118         int i, rc, lcount;
1119         AttrInfo *ai = (AttrInfo *)mc;
1120         mc = ai->ai_cursor;
1121
1122         dbi = ai->ai_dbi;
1123         for (i=0; keys[i].bv_val; i++) {
1124         itmp.kstr = keys[i];
1125         ic = tavl_find( (Avlnode *)ai->ai_root, &itmp, mdb_tool_idl_cmp );
1126
1127         /* No entry yet, create one */
1128         if ( !ic ) {
1129                 MDB_val key, data;
1130                 ID nid;
1131                 int rc;
1132
1133                 if ( ai->ai_clist ) {
1134                         ic = ai->ai_clist;
1135                         ai->ai_clist = ic->head;
1136                 } else {
1137                         ic = ch_malloc( sizeof( mdb_tool_idl_cache ) + itmp.kstr.bv_len + 4 );
1138                 }
1139                 ic->kstr.bv_len = itmp.kstr.bv_len;
1140                 ic->kstr.bv_val = (char *)(ic+1);
1141                 memcpy( ic->kstr.bv_val, itmp.kstr.bv_val, ic->kstr.bv_len );
1142                 ic->head = ic->tail = NULL;
1143                 ic->last = 0;
1144                 ic->count = 0;
1145                 ic->offset = 0;
1146                 ic->flags = 0;
1147                 tavl_insert( (Avlnode **)&ai->ai_root, ic, mdb_tool_idl_cmp,
1148                         avl_dup_error );
1149
1150                 /* load existing key count here */
1151                 key.mv_size = keys[i].bv_len;
1152                 key.mv_data = keys[i].bv_val;
1153                 rc = mdb_cursor_get( mc, &key, &data, MDB_SET );
1154                 if ( rc == 0 ) {
1155                         ic->flags |= WAS_FOUND;
1156                         nid = *(ID *)data.mv_data;
1157                         if ( nid == 0 ) {
1158                                 ic->count = MDB_IDL_DB_SIZE+1;
1159                                 ic->flags |= WAS_RANGE;
1160                         } else {
1161                                 size_t count;
1162
1163                                 mdb_cursor_count( mc, &count );
1164                                 ic->count = count;
1165                                 ic->first = nid;
1166                                 ic->offset = count & (IDBLOCK-1);
1167                         }
1168                 }
1169         }
1170         /* are we a range already? */
1171         if ( ic->count > MDB_IDL_DB_SIZE ) {
1172                 ic->last = id;
1173                 continue;
1174         /* Are we at the limit, and converting to a range? */
1175         } else if ( ic->count == MDB_IDL_DB_SIZE ) {
1176                 if ( ic->head ) {
1177                         ic->tail->next = ai->ai_flist;
1178                         ai->ai_flist = ic->head;
1179                 }
1180                 ic->head = ic->tail = NULL;
1181                 ic->last = id;
1182                 ic->count++;
1183                 continue;
1184         }
1185         /* No free block, create that too */
1186         lcount = ic->count & (IDBLOCK-1);
1187         if ( !ic->tail || lcount == 0) {
1188                 if ( ai->ai_flist ) {
1189                         ice = ai->ai_flist;
1190                         ai->ai_flist = ice->next;
1191                 } else {
1192                         ice = ch_malloc( sizeof( mdb_tool_idl_cache_entry ));
1193                 }
1194                 ice->next = NULL;
1195                 if ( !ic->head ) {
1196                         ic->head = ice;
1197                 } else {
1198                         ic->tail->next = ice;
1199                 }
1200                 ic->tail = ice;
1201                 if ( lcount )
1202                         ice->ids[lcount-1] = 0;
1203                 if ( !ic->count )
1204                         ic->first = id;
1205         }
1206         ice = ic->tail;
1207         if (!lcount || ice->ids[lcount-1] != id)
1208                 ice->ids[lcount] = id;
1209         ic->count++;
1210         }
1211
1212         return 0;
1213 }
1214 #endif /* MDB_TOOL_IDL_CACHING */