git.sur5r.net Git - openldap/blob - servers/slapd/back-bdb/tools.c

   1 /* tools.c - tools for slap tools */
   2 /* $OpenLDAP$ */
   3 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
   4  *
   5  * Copyright 2000-2017 The OpenLDAP Foundation.
   6  * All rights reserved.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted only as authorized by the OpenLDAP
  10  * Public License.
  11  *
  12  * A copy of this license is available in the file LICENSE in the
  13  * top-level directory of the distribution or, alternatively, at
  14  * <http://www.OpenLDAP.org/license.html>.
  15  */
  16
  17 #include "portable.h"
  18
  19 #include <stdio.h>
  20 #include <ac/string.h>
  21 #include <ac/errno.h>
  22
  23 #define AVL_INTERNAL
  24 #include "back-bdb.h"
  25 #include "idl.h"
  26
  27 static DBC *cursor = NULL;
  28 static DBT key, data;
  29 static EntryHeader eh;
  30 static ID nid, previd = NOID;
  31 static char ehbuf[16];
  32
  33 typedef struct dn_id {
  34         ID id;
  35         struct berval dn;
  36 } dn_id;
  37
  38 #define HOLE_SIZE       4096
  39 static dn_id hbuf[HOLE_SIZE], *holes = hbuf;
  40 static unsigned nhmax = HOLE_SIZE;
  41 static unsigned nholes;
  42
  43 static int index_nattrs;
  44
  45 static struct berval    *tool_base;
  46 static int              tool_scope;
  47 static Filter           *tool_filter;
  48 static Entry            *tool_next_entry;
  49
  50 #ifdef BDB_TOOL_IDL_CACHING
  51 #define bdb_tool_idl_cmp                BDB_SYMBOL(tool_idl_cmp)
  52 #define bdb_tool_idl_flush_one          BDB_SYMBOL(tool_idl_flush_one)
  53 #define bdb_tool_idl_flush              BDB_SYMBOL(tool_idl_flush)
  54
  55 static int bdb_tool_idl_flush( BackendDB *be );
  56
  57 #define IDBLOCK 1024
  58
  59 typedef struct bdb_tool_idl_cache_entry {
  60         struct bdb_tool_idl_cache_entry *next;
  61         ID ids[IDBLOCK];
  62 } bdb_tool_idl_cache_entry;
  63
  64 typedef struct bdb_tool_idl_cache {
  65         struct berval kstr;
  66         bdb_tool_idl_cache_entry *head, *tail;
  67         ID first, last;
  68         int count;
  69 } bdb_tool_idl_cache;
  70
  71 static bdb_tool_idl_cache_entry *bdb_tool_idl_free_list;
  72 #endif  /* BDB_TOOL_IDL_CACHING */
  73
  74 static ID bdb_tool_ix_id;
  75 static Operation *bdb_tool_ix_op;
  76 static int *bdb_tool_index_threads, bdb_tool_index_tcount;
  77 static void *bdb_tool_index_rec;
  78 static struct bdb_info *bdb_tool_info;
  79 static ldap_pvt_thread_mutex_t bdb_tool_index_mutex;
  80 static ldap_pvt_thread_cond_t bdb_tool_index_cond_main;
  81 static ldap_pvt_thread_cond_t bdb_tool_index_cond_work;
  82
  83 #if DB_VERSION_FULL >= 0x04060000
  84 #define USE_TRICKLE     1
  85 #else
  86 /* Seems to slow things down too much in BDB 4.5 */
  87 #undef USE_TRICKLE
  88 #endif
  89
  90 #ifdef USE_TRICKLE
  91 static ldap_pvt_thread_mutex_t bdb_tool_trickle_mutex;
  92 static ldap_pvt_thread_cond_t bdb_tool_trickle_cond;
  93 static ldap_pvt_thread_cond_t bdb_tool_trickle_cond_end;
  94
  95 static void * bdb_tool_trickle_task( void *ctx, void *ptr );
  96 static int bdb_tool_trickle_active;
  97 #endif
  98
  99 static void * bdb_tool_index_task( void *ctx, void *ptr );
 100
 101 static int
 102 bdb_tool_entry_get_int( BackendDB *be, ID id, Entry **ep );
 103
 104 static int bdb_tool_threads;
 105
 106 int bdb_tool_entry_open(
 107         BackendDB *be, int mode )
 108 {
 109         struct bdb_info *bdb = (struct bdb_info *) be->be_private;
 110
 111         /* initialize key and data thangs */
 112         DBTzero( &key );
 113         DBTzero( &data );
 114         key.flags = DB_DBT_USERMEM;
 115         key.data = &nid;
 116         key.size = key.ulen = sizeof( nid );
 117         data.flags = DB_DBT_USERMEM;
 118
 119         if (cursor == NULL) {
 120                 int rc = bdb->bi_id2entry->bdi_db->cursor(
 121                         bdb->bi_id2entry->bdi_db, bdb->bi_cache.c_txn, &cursor,
 122                         bdb->bi_db_opflags );
 123                 if( rc != 0 ) {
 124                         return -1;
 125                 }
 126         }
 127
 128         /* Set up for threaded slapindex */
 129         if (( slapMode & (SLAP_TOOL_QUICK|SLAP_TOOL_READONLY)) == SLAP_TOOL_QUICK ) {
 130                 if ( !bdb_tool_info ) {
 131 #ifdef USE_TRICKLE
 132                         ldap_pvt_thread_mutex_init( &bdb_tool_trickle_mutex );
 133                         ldap_pvt_thread_cond_init( &bdb_tool_trickle_cond );
 134                         ldap_pvt_thread_cond_init( &bdb_tool_trickle_cond_end );
 135                         ldap_pvt_thread_pool_submit( &connection_pool, bdb_tool_trickle_task, bdb->bi_dbenv );
 136 #endif
 137
 138                         ldap_pvt_thread_mutex_init( &bdb_tool_index_mutex );
 139                         ldap_pvt_thread_cond_init( &bdb_tool_index_cond_main );
 140                         ldap_pvt_thread_cond_init( &bdb_tool_index_cond_work );
 141                         if ( bdb->bi_nattrs ) {
 142                                 int i;
 143                                 bdb_tool_threads = slap_tool_thread_max - 1;
 144                                 if ( bdb_tool_threads > 1 ) {
 145                                         bdb_tool_index_threads = ch_malloc( bdb_tool_threads * sizeof( int ));
 146                                         bdb_tool_index_rec = ch_malloc( bdb->bi_nattrs * sizeof( IndexRec ));
 147                                         bdb_tool_index_tcount = bdb_tool_threads - 1;
 148                                         for (i=1; i<bdb_tool_threads; i++) {
 149                                                 int *ptr = ch_malloc( sizeof( int ));
 150                                                 *ptr = i;
 151                                                 ldap_pvt_thread_pool_submit( &connection_pool,
 152                                                         bdb_tool_index_task, ptr );
 153                                         }
 154                                 }
 155                         }
 156                         bdb_tool_info = bdb;
 157                 }
 158         }
 159
 160         return 0;
 161 }
 162
 163 int bdb_tool_entry_close(
 164         BackendDB *be )
 165 {
 166         if ( bdb_tool_info ) {
 167                 slapd_shutdown = 1;
 168 #ifdef USE_TRICKLE
 169                 ldap_pvt_thread_mutex_lock( &bdb_tool_trickle_mutex );
 170
 171                 /* trickle thread may not have started yet */
 172                 while ( !bdb_tool_trickle_active )
 173                         ldap_pvt_thread_cond_wait( &bdb_tool_trickle_cond_end,
 174                                         &bdb_tool_trickle_mutex );
 175
 176                 ldap_pvt_thread_cond_signal( &bdb_tool_trickle_cond );
 177                 while ( bdb_tool_trickle_active )
 178                         ldap_pvt_thread_cond_wait( &bdb_tool_trickle_cond_end,
 179                                         &bdb_tool_trickle_mutex );
 180                 ldap_pvt_thread_mutex_unlock( &bdb_tool_trickle_mutex );
 181 #endif
 182                 if ( bdb_tool_threads > 1 ) {
 183                         ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex );
 184
 185                         /* There might still be some threads starting */
 186                         while ( bdb_tool_index_tcount > 0 ) {
 187                                 ldap_pvt_thread_cond_wait( &bdb_tool_index_cond_main,
 188                                                 &bdb_tool_index_mutex );
 189                         }
 190
 191                         bdb_tool_index_tcount = bdb_tool_threads - 1;
 192                         ldap_pvt_thread_cond_broadcast( &bdb_tool_index_cond_work );
 193
 194                         /* Make sure all threads are stopped */
 195                         while ( bdb_tool_index_tcount > 0 ) {
 196                                 ldap_pvt_thread_cond_wait( &bdb_tool_index_cond_main,
 197                                         &bdb_tool_index_mutex );
 198                         }
 199                         ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex );
 200
 201                         ch_free( bdb_tool_index_threads );
 202                         ch_free( bdb_tool_index_rec );
 203                         bdb_tool_index_tcount = bdb_tool_threads - 1;
 204                 }
 205                 bdb_tool_info = NULL;
 206                 slapd_shutdown = 0;
 207         }
 208
 209         if( eh.bv.bv_val ) {
 210                 ch_free( eh.bv.bv_val );
 211                 eh.bv.bv_val = NULL;
 212         }
 213
 214         if( cursor ) {
 215                 cursor->c_close( cursor );
 216                 cursor = NULL;
 217         }
 218
 219 #ifdef BDB_TOOL_IDL_CACHING
 220         bdb_tool_idl_flush( be );
 221 #endif
 222
 223         if( nholes ) {
 224                 unsigned i;
 225                 fprintf( stderr, "Error, entries missing!\n");
 226                 for (i=0; i<nholes; i++) {
 227                         fprintf(stderr, "  entry %ld: %s\n",
 228                                 holes[i].id, holes[i].dn.bv_val);
 229                 }
 230                 return -1;
 231         }
 232
 233         return 0;
 234 }
 235
 236 ID
 237 bdb_tool_entry_first_x(
 238         BackendDB *be,
 239         struct berval *base,
 240         int scope,
 241         Filter *f )
 242 {
 243         tool_base = base;
 244         tool_scope = scope;
 245         tool_filter = f;
 246
 247         return bdb_tool_entry_next( be );
 248 }
 249
 250 ID bdb_tool_entry_next(
 251         BackendDB *be )
 252 {
 253         int rc;
 254         ID id;
 255         struct bdb_info *bdb;
 256
 257         assert( be != NULL );
 258         assert( slapMode & SLAP_TOOL_MODE );
 259
 260         bdb = (struct bdb_info *) be->be_private;
 261         assert( bdb != NULL );
 262
 263 next:;
 264         /* Get the header */
 265         data.ulen = data.dlen = sizeof( ehbuf );
 266         data.data = ehbuf;
 267         data.flags |= DB_DBT_PARTIAL;
 268         rc = cursor->c_get( cursor, &key, &data, DB_NEXT );
 269
 270         if( rc ) {
 271                 /* If we're doing linear indexing and there are more attrs to
 272                  * index, and we're at the end of the database, start over.
 273                  */
 274                 if ( index_nattrs && rc == DB_NOTFOUND ) {
 275                         /* optional - do a checkpoint here? */
 276                         bdb_attr_info_free( bdb->bi_attrs[0] );
 277                         bdb->bi_attrs[0] = bdb->bi_attrs[index_nattrs];
 278                         index_nattrs--;
 279                         rc = cursor->c_get( cursor, &key, &data, DB_FIRST );
 280                         if ( rc ) {
 281                                 return NOID;
 282                         }
 283                 } else {
 284                         return NOID;
 285                 }
 286         }
 287
 288         BDB_DISK2ID( key.data, &id );
 289         previd = id;
 290
 291         if ( tool_filter || tool_base ) {
 292                 static Operation op = {0};
 293                 static Opheader ohdr = {0};
 294
 295                 op.o_hdr = &ohdr;
 296                 op.o_bd = be;
 297                 op.o_tmpmemctx = NULL;
 298                 op.o_tmpmfuncs = &ch_mfuncs;
 299
 300                 if ( tool_next_entry ) {
 301                         bdb_entry_release( &op, tool_next_entry, 0 );
 302                         tool_next_entry = NULL;
 303                 }
 304
 305                 rc = bdb_tool_entry_get_int( be, id, &tool_next_entry );
 306                 if ( rc == LDAP_NO_SUCH_OBJECT ) {
 307                         goto next;
 308                 }
 309
 310                 assert( tool_next_entry != NULL );
 311
 312 #ifdef BDB_HIER
 313                 /* TODO: needed until BDB_HIER is handled accordingly
 314                  * in bdb_tool_entry_get_int() */
 315                 if ( tool_base && !dnIsSuffixScope( &tool_next_entry->e_nname, tool_base, tool_scope ) )
 316                 {
 317                         bdb_entry_release( &op, tool_next_entry, 0 );
 318                         tool_next_entry = NULL;
 319                         goto next;
 320                 }
 321 #endif
 322
 323                 if ( tool_filter && test_filter( NULL, tool_next_entry, tool_filter ) != LDAP_COMPARE_TRUE )
 324                 {
 325                         bdb_entry_release( &op, tool_next_entry, 0 );
 326                         tool_next_entry = NULL;
 327                         goto next;
 328                 }
 329         }
 330
 331         return id;
 332 }
 333
 334 ID bdb_tool_dn2id_get(
 335         Backend *be,
 336         struct berval *dn
 337 )
 338 {
 339         Operation op = {0};
 340         Opheader ohdr = {0};
 341         EntryInfo *ei = NULL;
 342         int rc;
 343
 344         if ( BER_BVISEMPTY(dn) )
 345                 return 0;
 346
 347         op.o_hdr = &ohdr;
 348         op.o_bd = be;
 349         op.o_tmpmemctx = NULL;
 350         op.o_tmpmfuncs = &ch_mfuncs;
 351
 352         rc = bdb_cache_find_ndn( &op, 0, dn, &ei );
 353         if ( ei ) bdb_cache_entryinfo_unlock( ei );
 354         if ( rc == DB_NOTFOUND )
 355                 return NOID;
 356
 357         return ei->bei_id;
 358 }
 359
 360 static int
 361 bdb_tool_entry_get_int( BackendDB *be, ID id, Entry **ep )
 362 {
 363         Entry *e = NULL;
 364         char *dptr;
 365         int rc, eoff;
 366
 367         assert( be != NULL );
 368         assert( slapMode & SLAP_TOOL_MODE );
 369
 370         if ( ( tool_filter || tool_base ) && id == previd && tool_next_entry != NULL ) {
 371                 *ep = tool_next_entry;
 372                 tool_next_entry = NULL;
 373                 return LDAP_SUCCESS;
 374         }
 375
 376         if ( id != previd ) {
 377                 data.ulen = data.dlen = sizeof( ehbuf );
 378                 data.data = ehbuf;
 379                 data.flags |= DB_DBT_PARTIAL;
 380
 381                 BDB_ID2DISK( id, &nid );
 382                 rc = cursor->c_get( cursor, &key, &data, DB_SET );
 383                 if ( rc ) {
 384                         rc = LDAP_OTHER;
 385                         goto done;
 386                 }
 387         }
 388
 389         /* Get the header */
 390         dptr = eh.bv.bv_val;
 391         eh.bv.bv_val = ehbuf;
 392         eh.bv.bv_len = data.size;
 393         rc = entry_header( &eh );
 394         eoff = eh.data - eh.bv.bv_val;
 395         eh.bv.bv_val = dptr;
 396         if ( rc ) {
 397                 rc = LDAP_OTHER;
 398                 goto done;
 399         }
 400
 401         /* Get the size */
 402         data.flags &= ~DB_DBT_PARTIAL;
 403         data.ulen = 0;
 404         rc = cursor->c_get( cursor, &key, &data, DB_CURRENT );
 405         if ( rc != DB_BUFFER_SMALL ) {
 406                 rc = LDAP_OTHER;
 407                 goto done;
 408         }
 409
 410         /* Allocate a block and retrieve the data */
 411         eh.bv.bv_len = eh.nvals * sizeof( struct berval ) + data.size;
 412         eh.bv.bv_val = ch_realloc( eh.bv.bv_val, eh.bv.bv_len );
 413         eh.data = eh.bv.bv_val + eh.nvals * sizeof( struct berval );
 414         data.data = eh.data;
 415         data.ulen = data.size;
 416
 417         /* Skip past already parsed nattr/nvals */
 418         eh.data += eoff;
 419
 420         rc = cursor->c_get( cursor, &key, &data, DB_CURRENT );
 421         if ( rc ) {
 422                 rc = LDAP_OTHER;
 423                 goto done;
 424         }
 425
 426 #ifndef BDB_HIER
 427         /* TODO: handle BDB_HIER accordingly */
 428         if ( tool_base != NULL ) {
 429                 struct berval ndn;
 430                 entry_decode_dn( &eh, NULL, &ndn );
 431
 432                 if ( !dnIsSuffixScope( &ndn, tool_base, tool_scope ) ) {
 433                         return LDAP_NO_SUCH_OBJECT;
 434                 }
 435         }
 436 #endif
 437
 438 #ifdef SLAP_ZONE_ALLOC
 439         /* FIXME: will add ctx later */
 440         rc = entry_decode( &eh, &e, NULL );
 441 #else
 442         rc = entry_decode( &eh, &e );
 443 #endif
 444
 445         if( rc == LDAP_SUCCESS ) {
 446                 e->e_id = id;
 447 #ifdef BDB_HIER
 448                 if ( slapMode & SLAP_TOOL_READONLY ) {
 449                         struct bdb_info *bdb = (struct bdb_info *) be->be_private;
 450                         EntryInfo *ei = NULL;
 451                         Operation op = {0};
 452                         Opheader ohdr = {0};
 453
 454                         op.o_hdr = &ohdr;
 455                         op.o_bd = be;
 456                         op.o_tmpmemctx = NULL;
 457                         op.o_tmpmfuncs = &ch_mfuncs;
 458
 459                         rc = bdb_cache_find_parent( &op, bdb->bi_cache.c_txn, id, &ei );
 460                         if ( rc == LDAP_SUCCESS ) {
 461                                 bdb_cache_entryinfo_unlock( ei );
 462                                 e->e_private = ei;
 463                                 ei->bei_e = e;
 464                                 bdb_fix_dn( e, 0 );
 465                                 ei->bei_e = NULL;
 466                                 e->e_private = NULL;
 467                         }
 468                 }
 469 #endif
 470         }
 471 done:
 472         if ( e != NULL ) {
 473                 *ep = e;
 474         }
 475
 476         return rc;
 477 }
 478
 479 Entry*
 480 bdb_tool_entry_get( BackendDB *be, ID id )
 481 {
 482         Entry *e = NULL;
 483
 484         (void)bdb_tool_entry_get_int( be, id, &e );
 485         return e;
 486 }
 487
 488 static int bdb_tool_next_id(
 489         Operation *op,
 490         DB_TXN *tid,
 491         Entry *e,
 492         struct berval *text,
 493         int hole )
 494 {
 495         struct berval dn = e->e_name;
 496         struct berval ndn = e->e_nname;
 497         struct berval pdn, npdn;
 498         EntryInfo *ei = NULL, eidummy;
 499         int rc;
 500
 501         if (ndn.bv_len == 0) {
 502                 e->e_id = 0;
 503                 return 0;
 504         }
 505
 506         rc = bdb_cache_find_ndn( op, tid, &ndn, &ei );
 507         if ( ei ) bdb_cache_entryinfo_unlock( ei );
 508         if ( rc == DB_NOTFOUND ) {
 509                 if ( !be_issuffix( op->o_bd, &ndn ) ) {
 510                         ID eid = e->e_id;
 511                         dnParent( &dn, &pdn );
 512                         dnParent( &ndn, &npdn );
 513                         e->e_name = pdn;
 514                         e->e_nname = npdn;
 515                         rc = bdb_tool_next_id( op, tid, e, text, 1 );
 516                         e->e_name = dn;
 517                         e->e_nname = ndn;
 518                         if ( rc ) {
 519                                 return rc;
 520                         }
 521                         /* If parent didn't exist, it was created just now
 522                          * and its ID is now in e->e_id. Make sure the current
 523                          * entry gets added under the new parent ID.
 524                          */
 525                         if ( eid != e->e_id ) {
 526                                 eidummy.bei_id = e->e_id;
 527                                 ei = &eidummy;
 528                         }
 529                 }
 530                 rc = bdb_next_id( op->o_bd, &e->e_id );
 531                 if ( rc ) {
 532                         snprintf( text->bv_val, text->bv_len,
 533                                 "next_id failed: %s (%d)",
 534                                 db_strerror(rc), rc );
 535                 Debug( LDAP_DEBUG_ANY,
 536                         "=> bdb_tool_next_id: %s\n", text->bv_val, 0, 0 );
 537                         return rc;
 538                 }
 539                 rc = bdb_dn2id_add( op, tid, ei, e );
 540                 if ( rc ) {
 541                         snprintf( text->bv_val, text->bv_len,
 542                                 "dn2id_add failed: %s (%d)",
 543                                 db_strerror(rc), rc );
 544                 Debug( LDAP_DEBUG_ANY,
 545                         "=> bdb_tool_next_id: %s\n", text->bv_val, 0, 0 );
 546                 } else if ( hole ) {
 547                         if ( nholes == nhmax - 1 ) {
 548                                 if ( holes == hbuf ) {
 549                                         holes = ch_malloc( nhmax * sizeof(dn_id) * 2 );
 550                                         AC_MEMCPY( holes, hbuf, sizeof(hbuf) );
 551                                 } else {
 552                                         holes = ch_realloc( holes, nhmax * sizeof(dn_id) * 2 );
 553                                 }
 554                                 nhmax *= 2;
 555                         }
 556                         ber_dupbv( &holes[nholes].dn, &ndn );
 557                         holes[nholes++].id = e->e_id;
 558                 }
 559         } else if ( !hole ) {
 560                 unsigned i, j;
 561
 562                 e->e_id = ei->bei_id;
 563
 564                 for ( i=0; i<nholes; i++) {
 565                         if ( holes[i].id == e->e_id ) {
 566                                 free(holes[i].dn.bv_val);
 567                                 for (j=i;j<nholes;j++) holes[j] = holes[j+1];
 568                                 holes[j].id = 0;
 569                                 nholes--;
 570                                 break;
 571                         } else if ( holes[i].id > e->e_id ) {
 572                                 break;
 573                         }
 574                 }
 575         }
 576         return rc;
 577 }
 578
 579 static int
 580 bdb_tool_index_add(
 581         Operation *op,
 582         DB_TXN *txn,
 583         Entry *e )
 584 {
 585         struct bdb_info *bdb = (struct bdb_info *) op->o_bd->be_private;
 586
 587         if ( !bdb->bi_nattrs )
 588                 return 0;
 589
 590         if ( bdb_tool_threads > 1 ) {
 591                 IndexRec *ir;
 592                 int i, rc;
 593                 Attribute *a;
 594
 595                 ir = bdb_tool_index_rec;
 596                 memset(ir, 0, bdb->bi_nattrs * sizeof( IndexRec ));
 597
 598                 for ( a = e->e_attrs; a != NULL; a = a->a_next ) {
 599                         rc = bdb_index_recset( bdb, a, a->a_desc->ad_type,
 600                                 &a->a_desc->ad_tags, ir );
 601                         if ( rc )
 602                                 return rc;
 603                 }
 604                 bdb_tool_ix_id = e->e_id;
 605                 bdb_tool_ix_op = op;
 606                 ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex );
 607                 /* Wait for all threads to be ready */
 608                 while ( bdb_tool_index_tcount > 0 ) {
 609                         ldap_pvt_thread_cond_wait( &bdb_tool_index_cond_main,
 610                                 &bdb_tool_index_mutex );
 611                 }
 612                 for ( i=1; i<bdb_tool_threads; i++ )
 613                         bdb_tool_index_threads[i] = LDAP_BUSY;
 614                 bdb_tool_index_tcount = bdb_tool_threads - 1;
 615                 ldap_pvt_thread_cond_broadcast( &bdb_tool_index_cond_work );
 616                 ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex );
 617                 rc = bdb_index_recrun( op, bdb, ir, e->e_id, 0 );
 618                 if ( rc )
 619                         return rc;
 620                 ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex );
 621                 for ( i=1; i<bdb_tool_threads; i++ ) {
 622                         if ( bdb_tool_index_threads[i] == LDAP_BUSY ) {
 623                                 ldap_pvt_thread_cond_wait( &bdb_tool_index_cond_main,
 624                                         &bdb_tool_index_mutex );
 625                                 i--;
 626                                 continue;
 627                         }
 628                         if ( bdb_tool_index_threads[i] ) {
 629                                 rc = bdb_tool_index_threads[i];
 630                                 break;
 631                         }
 632                 }
 633                 ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex );
 634                 return rc;
 635         } else {
 636                 return bdb_index_entry_add( op, txn, e );
 637         }
 638 }
 639
 640 ID bdb_tool_entry_put(
 641         BackendDB *be,
 642         Entry *e,
 643         struct berval *text )
 644 {
 645         int rc;
 646         struct bdb_info *bdb;
 647         DB_TXN *tid = NULL;
 648         Operation op = {0};
 649         Opheader ohdr = {0};
 650
 651         assert( be != NULL );
 652         assert( slapMode & SLAP_TOOL_MODE );
 653
 654         assert( text != NULL );
 655         assert( text->bv_val != NULL );
 656         assert( text->bv_val[0] == '\0' );      /* overconservative? */
 657
 658         Debug( LDAP_DEBUG_TRACE, "=> " LDAP_XSTRING(bdb_tool_entry_put)
 659                 "( %ld, \"%s\" )\n", (long) e->e_id, e->e_dn, 0 );
 660
 661         bdb = (struct bdb_info *) be->be_private;
 662
 663         if (! (slapMode & SLAP_TOOL_QUICK)) {
 664         rc = TXN_BEGIN( bdb->bi_dbenv, NULL, &tid,
 665                 bdb->bi_db_opflags );
 666         if( rc != 0 ) {
 667                 snprintf( text->bv_val, text->bv_len,
 668                         "txn_begin failed: %s (%d)",
 669                         db_strerror(rc), rc );
 670                 Debug( LDAP_DEBUG_ANY,
 671                         "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
 672                          text->bv_val, 0, 0 );
 673                 return NOID;
 674         }
 675         Debug( LDAP_DEBUG_TRACE, LDAP_XSTRING(bdb_tool_entry_put) ": txn id: %x\n",
 676                 tid->id(tid), 0, 0 );
 677         }
 678
 679         op.o_hdr = &ohdr;
 680         op.o_bd = be;
 681         op.o_tmpmemctx = NULL;
 682         op.o_tmpmfuncs = &ch_mfuncs;
 683
 684         /* add dn2id indices */
 685         rc = bdb_tool_next_id( &op, tid, e, text, 0 );
 686         if( rc != 0 ) {
 687                 goto done;
 688         }
 689
 690 #ifdef USE_TRICKLE
 691         if (( slapMode & SLAP_TOOL_QUICK ) && (( e->e_id & 0xfff ) == 0xfff )) {
 692                 ldap_pvt_thread_cond_signal( &bdb_tool_trickle_cond );
 693         }
 694 #endif
 695
 696         if ( !bdb->bi_linear_index )
 697                 rc = bdb_tool_index_add( &op, tid, e );
 698         if( rc != 0 ) {
 699                 snprintf( text->bv_val, text->bv_len,
 700                                 "index_entry_add failed: %s (%d)",
 701                                 rc == LDAP_OTHER ? "Internal error" :
 702                                 db_strerror(rc), rc );
 703                 Debug( LDAP_DEBUG_ANY,
 704                         "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
 705                         text->bv_val, 0, 0 );
 706                 goto done;
 707         }
 708
 709         /* id2entry index */
 710         rc = bdb_id2entry_add( be, tid, e );
 711         if( rc != 0 ) {
 712                 snprintf( text->bv_val, text->bv_len,
 713                                 "id2entry_add failed: %s (%d)",
 714                                 db_strerror(rc), rc );
 715                 Debug( LDAP_DEBUG_ANY,
 716                         "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
 717                         text->bv_val, 0, 0 );
 718                 goto done;
 719         }
 720
 721 done:
 722         if( rc == 0 ) {
 723                 if ( !( slapMode & SLAP_TOOL_QUICK )) {
 724                 rc = TXN_COMMIT( tid, 0 );
 725                 if( rc != 0 ) {
 726                         snprintf( text->bv_val, text->bv_len,
 727                                         "txn_commit failed: %s (%d)",
 728                                         db_strerror(rc), rc );
 729                         Debug( LDAP_DEBUG_ANY,
 730                                 "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
 731                                 text->bv_val, 0, 0 );
 732                         e->e_id = NOID;
 733                 }
 734                 }
 735
 736         } else {
 737                 if ( !( slapMode & SLAP_TOOL_QUICK )) {
 738                 TXN_ABORT( tid );
 739                 snprintf( text->bv_val, text->bv_len,
 740                         "txn_aborted! %s (%d)",
 741                         rc == LDAP_OTHER ? "Internal error" :
 742                         db_strerror(rc), rc );
 743                 Debug( LDAP_DEBUG_ANY,
 744                         "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
 745                         text->bv_val, 0, 0 );
 746                 }
 747                 e->e_id = NOID;
 748         }
 749
 750         if ( cursor == NULL )
 751         {
 752                 int rc = bdb->bi_id2entry->bdi_db->cursor(
 753                         bdb->bi_id2entry->bdi_db, bdb->bi_cache.c_txn, &cursor,
 754                         bdb->bi_db_opflags );
 755                 if ( rc != 0 )
 756                         e->e_id = NOID;
 757         }
 758
 759         return e->e_id;
 760 }
 761
 762 int bdb_tool_entry_reindex(
 763         BackendDB *be,
 764         ID id,
 765         AttributeDescription **adv )
 766 {
 767         struct bdb_info *bi = (struct bdb_info *) be->be_private;
 768         int rc;
 769         Entry *e;
 770         DB_TXN *tid = NULL;
 771         Operation op = {0};
 772         Opheader ohdr = {0};
 773
 774         Debug( LDAP_DEBUG_ARGS,
 775                 "=> " LDAP_XSTRING(bdb_tool_entry_reindex) "( %ld )\n",
 776                 (long) id, 0, 0 );
 777         assert( tool_base == NULL );
 778         assert( tool_filter == NULL );
 779
 780         /* No indexes configured, nothing to do. Could return an
 781          * error here to shortcut things.
 782          */
 783         if (!bi->bi_attrs) {
 784                 return 0;
 785         }
 786
 787         /* Check for explicit list of attrs to index */
 788         if ( adv ) {
 789                 int i, j, n;
 790
 791                 if ( bi->bi_attrs[0]->ai_desc != adv[0] ) {
 792                         /* count */
 793                         for ( n = 0; adv[n]; n++ ) ;
 794
 795                         /* insertion sort */
 796                         for ( i = 0; i < n; i++ ) {
 797                                 AttributeDescription *ad = adv[i];
 798                                 for ( j = i-1; j>=0; j--) {
 799                                         if ( SLAP_PTRCMP( adv[j], ad ) <= 0 ) break;
 800                                         adv[j+1] = adv[j];
 801                                 }
 802                                 adv[j+1] = ad;
 803                         }
 804                 }
 805
 806                 for ( i = 0; adv[i]; i++ ) {
 807                         if ( bi->bi_attrs[i]->ai_desc != adv[i] ) {
 808                                 for ( j = i+1; j < bi->bi_nattrs; j++ ) {
 809                                         if ( bi->bi_attrs[j]->ai_desc == adv[i] ) {
 810                                                 AttrInfo *ai = bi->bi_attrs[i];
 811                                                 bi->bi_attrs[i] = bi->bi_attrs[j];
 812                                                 bi->bi_attrs[j] = ai;
 813                                                 break;
 814                                         }
 815                                 }
 816                                 if ( j == bi->bi_nattrs ) {
 817                                         Debug( LDAP_DEBUG_ANY,
 818                                                 LDAP_XSTRING(bdb_tool_entry_reindex)
 819                                                 ": no index configured for %s\n",
 820                                                 adv[i]->ad_cname.bv_val, 0, 0 );
 821                                         return -1;
 822                                 }
 823                         }
 824                 }
 825                 bi->bi_nattrs = i;
 826         }
 827
 828         /* Get the first attribute to index */
 829         if (bi->bi_linear_index && !index_nattrs) {
 830                 index_nattrs = bi->bi_nattrs - 1;
 831                 bi->bi_nattrs = 1;
 832         }
 833
 834         e = bdb_tool_entry_get( be, id );
 835
 836         if( e == NULL ) {
 837                 Debug( LDAP_DEBUG_ANY,
 838                         LDAP_XSTRING(bdb_tool_entry_reindex)
 839                         ": could not locate id=%ld\n",
 840                         (long) id, 0, 0 );
 841                 return -1;
 842         }
 843
 844         op.o_hdr = &ohdr;
 845         op.o_bd = be;
 846         op.o_tmpmemctx = NULL;
 847         op.o_tmpmfuncs = &ch_mfuncs;
 848
 849         if (! (slapMode & SLAP_TOOL_QUICK)) {
 850         rc = TXN_BEGIN( bi->bi_dbenv, NULL, &tid, bi->bi_db_opflags );
 851         if( rc != 0 ) {
 852                 Debug( LDAP_DEBUG_ANY,
 853                         "=> " LDAP_XSTRING(bdb_tool_entry_reindex) ": "
 854                         "txn_begin failed: %s (%d)\n",
 855                         db_strerror(rc), rc, 0 );
 856                 goto done;
 857         }
 858         Debug( LDAP_DEBUG_TRACE, LDAP_XSTRING(bdb_tool_entry_reindex) ": txn id: %x\n",
 859                 tid->id(tid), 0, 0 );
 860         }
 861
 862         /*
 863          * just (re)add them for now
 864          * assume that some other routine (not yet implemented)
 865          * will zap index databases
 866          *
 867          */
 868
 869         Debug( LDAP_DEBUG_TRACE,
 870                 "=> " LDAP_XSTRING(bdb_tool_entry_reindex) "( %ld, \"%s\" )\n",
 871                 (long) id, e->e_dn, 0 );
 872
 873         rc = bdb_tool_index_add( &op, tid, e );
 874
 875 done:
 876         if( rc == 0 ) {
 877                 if (! (slapMode & SLAP_TOOL_QUICK)) {
 878                 rc = TXN_COMMIT( tid, 0 );
 879                 if( rc != 0 ) {
 880                         Debug( LDAP_DEBUG_ANY,
 881                                 "=> " LDAP_XSTRING(bdb_tool_entry_reindex)
 882                                 ": txn_commit failed: %s (%d)\n",
 883                                 db_strerror(rc), rc, 0 );
 884                         e->e_id = NOID;
 885                 }
 886                 }
 887
 888         } else {
 889                 if (! (slapMode & SLAP_TOOL_QUICK)) {
 890                 TXN_ABORT( tid );
 891                 Debug( LDAP_DEBUG_ANY,
 892                         "=> " LDAP_XSTRING(bdb_tool_entry_reindex)
 893                         ": txn_aborted! %s (%d)\n",
 894                         db_strerror(rc), rc, 0 );
 895                 }
 896                 e->e_id = NOID;
 897         }
 898         bdb_entry_release( &op, e, 0 );
 899
 900         return rc;
 901 }
 902
 903 ID bdb_tool_entry_modify(
 904         BackendDB *be,
 905         Entry *e,
 906         struct berval *text )
 907 {
 908         int rc;
 909         struct bdb_info *bdb;
 910         DB_TXN *tid = NULL;
 911         Operation op = {0};
 912         Opheader ohdr = {0};
 913
 914         assert( be != NULL );
 915         assert( slapMode & SLAP_TOOL_MODE );
 916
 917         assert( text != NULL );
 918         assert( text->bv_val != NULL );
 919         assert( text->bv_val[0] == '\0' );      /* overconservative? */
 920
 921         assert ( e->e_id != NOID );
 922
 923         Debug( LDAP_DEBUG_TRACE,
 924                 "=> " LDAP_XSTRING(bdb_tool_entry_modify) "( %ld, \"%s\" )\n",
 925                 (long) e->e_id, e->e_dn, 0 );
 926
 927         bdb = (struct bdb_info *) be->be_private;
 928
 929         if (! (slapMode & SLAP_TOOL_QUICK)) {
 930                 if( cursor ) {
 931                         cursor->c_close( cursor );
 932                         cursor = NULL;
 933                 }
 934                 rc = TXN_BEGIN( bdb->bi_dbenv, NULL, &tid,
 935                         bdb->bi_db_opflags );
 936                 if( rc != 0 ) {
 937                         snprintf( text->bv_val, text->bv_len,
 938                                 "txn_begin failed: %s (%d)",
 939                                 db_strerror(rc), rc );
 940                         Debug( LDAP_DEBUG_ANY,
 941                                 "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n",
 942                                  text->bv_val, 0, 0 );
 943                         return NOID;
 944                 }
 945                 Debug( LDAP_DEBUG_TRACE, LDAP_XSTRING(bdb_tool_entry_modify) ": txn id: %x\n",
 946                         tid->id(tid), 0, 0 );
 947         }
 948
 949         op.o_hdr = &ohdr;
 950         op.o_bd = be;
 951         op.o_tmpmemctx = NULL;
 952         op.o_tmpmfuncs = &ch_mfuncs;
 953
 954         /* id2entry index */
 955         rc = bdb_id2entry_update( be, tid, e );
 956         if( rc != 0 ) {
 957                 snprintf( text->bv_val, text->bv_len,
 958                                 "id2entry_add failed: %s (%d)",
 959                                 db_strerror(rc), rc );
 960                 Debug( LDAP_DEBUG_ANY,
 961                         "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n",
 962                         text->bv_val, 0, 0 );
 963                 goto done;
 964         }
 965
 966 done:
 967         if( rc == 0 ) {
 968                 if (! (slapMode & SLAP_TOOL_QUICK)) {
 969                 rc = TXN_COMMIT( tid, 0 );
 970                 if( rc != 0 ) {
 971                         snprintf( text->bv_val, text->bv_len,
 972                                         "txn_commit failed: %s (%d)",
 973                                         db_strerror(rc), rc );
 974                         Debug( LDAP_DEBUG_ANY,
 975                                 "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": "
 976                                 "%s\n", text->bv_val, 0, 0 );
 977                         e->e_id = NOID;
 978                 }
 979                 }
 980
 981         } else {
 982                 if (! (slapMode & SLAP_TOOL_QUICK)) {
 983                 TXN_ABORT( tid );
 984                 snprintf( text->bv_val, text->bv_len,
 985                         "txn_aborted! %s (%d)",
 986                         db_strerror(rc), rc );
 987                 Debug( LDAP_DEBUG_ANY,
 988                         "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n",
 989                         text->bv_val, 0, 0 );
 990                 }
 991                 e->e_id = NOID;
 992         }
 993
 994         if ( cursor == NULL )
 995         {
 996                 int rc = bdb->bi_id2entry->bdi_db->cursor(
 997                         bdb->bi_id2entry->bdi_db, bdb->bi_cache.c_txn, &cursor,
 998                         bdb->bi_db_opflags );
 999                 if ( rc != 0 )
1000                         e->e_id = NOID;
1001         }
1002
1003         return e->e_id;
1004 }
1005
1006 int bdb_tool_entry_delete(
1007         BackendDB *be,
1008         struct berval *ndn,
1009         struct berval *text )
1010 {
1011         int rc;
1012         struct bdb_info *bdb;
1013         DB_TXN *tid = NULL;
1014         Operation op = {0};
1015         Opheader ohdr = {0};
1016         EntryInfo *ei, *eip;
1017         Entry *e;
1018         DB_LOCK lock;
1019
1020         assert( be != NULL );
1021         assert( slapMode & SLAP_TOOL_MODE );
1022
1023         assert( text != NULL );
1024         assert( text->bv_val != NULL );
1025         assert( text->bv_val[0] == '\0' );      /* overconservative? */
1026
1027         assert ( ndn != NULL );
1028         assert ( ndn->bv_val != NULL );
1029
1030         Debug( LDAP_DEBUG_TRACE,
1031                 "=> " LDAP_XSTRING(bdb_tool_entry_delete) "( %s )\n",
1032                 ndn->bv_val, 0, 0 );
1033
1034         bdb = (struct bdb_info *) be->be_private;
1035
1036         if (! (slapMode & SLAP_TOOL_QUICK)) {
1037                 if( cursor ) {
1038                         cursor->c_close( cursor );
1039                         cursor = NULL;
1040                 }
1041                 rc = TXN_BEGIN( bdb->bi_dbenv, NULL, &tid,
1042                         bdb->bi_db_opflags );
1043                 if( rc != 0 ) {
1044                         snprintf( text->bv_val, text->bv_len,
1045                                 "txn_begin failed: %s (%d)",
1046                                 db_strerror(rc), rc );
1047                         Debug( LDAP_DEBUG_ANY,
1048                                 "=> " LDAP_XSTRING(bdb_tool_entry_delete) ": %s\n",
1049                                  text->bv_val, 0, 0 );
1050                         return LDAP_OTHER;
1051                 }
1052                 rc = bdb->bi_id2entry->bdi_db->cursor(
1053                         bdb->bi_id2entry->bdi_db, bdb->bi_cache.c_txn, &cursor,
1054                         bdb->bi_db_opflags );
1055         }
1056
1057         op.o_hdr = &ohdr;
1058         op.o_bd = be;
1059         op.o_tmpmemctx = NULL;
1060         op.o_tmpmfuncs = &ch_mfuncs;
1061
1062         /* do the deletion */
1063         rc = bdb_dn2entry( &op, tid, ndn, &ei, 1, &lock );
1064         if( rc != 0 ) {
1065                 snprintf( text->bv_val, text->bv_len,
1066                         "dn2entry failed: %s (%d)",
1067                         db_strerror(rc), rc );
1068                 Debug( LDAP_DEBUG_ANY,
1069                         "=> " LDAP_XSTRING(bdb_tool_entry_delete) ": %s\n",
1070                                 text->bv_val, 0, 0 );
1071                 goto done;
1072         }
1073
1074         e = ei->bei_e;
1075         eip = ei->bei_parent;
1076
1077         rc = bdb_cache_children( &op, tid, e );
1078         if( rc != DB_NOTFOUND ) {
1079                 switch( rc ) {
1080                 case 0:
1081                         snprintf( text->bv_val, text->bv_len,
1082                                 "delete failed:"
1083                                 " subordinate objects must be deleted first");
1084                         break;
1085                 default:
1086                         snprintf( text->bv_val, text->bv_len,
1087                                 "has_children failed: %s (%d)",
1088                                 db_strerror(rc), rc );
1089                         break;
1090                 }
1091                 rc = -1;
1092                 Debug( LDAP_DEBUG_ANY,
1093                         "=> " LDAP_XSTRING(mdb_tool_entry_delete) ": %s\n",
1094                          text->bv_val, 0, 0 );
1095                 goto done;
1096         }
1097         rc = bdb_dn2id_delete( &op, tid, eip, e );
1098         if( rc != 0 ) {
1099                 snprintf( text->bv_val, text->bv_len,
1100                         "dn2entry failed: %s (%d)",
1101                         db_strerror(rc), rc );
1102                 Debug( LDAP_DEBUG_ANY,
1103                         "=> " LDAP_XSTRING(bdb_tool_entry_delete) ": %s\n",
1104                                 text->bv_val, 0, 0 );
1105                 goto done;
1106         }
1107
1108         rc = bdb_index_entry_del( &op, tid, e );
1109         if( rc != 0 ) {
1110                 snprintf( text->bv_val, text->bv_len,
1111                         "dn2entry failed: %s (%d)",
1112                         db_strerror(rc), rc );
1113                 Debug( LDAP_DEBUG_ANY,
1114                         "=> " LDAP_XSTRING(bdb_tool_entry_delete) ": %s\n",
1115                                 text->bv_val, 0, 0 );
1116                 goto done;
1117         }
1118
1119         rc = bdb_id2entry_delete( be, tid, e );
1120         if( rc != 0 ) {
1121                 snprintf( text->bv_val, text->bv_len,
1122                         "dn2entry failed: %s (%d)",
1123                         db_strerror(rc), rc );
1124                 Debug( LDAP_DEBUG_ANY,
1125                         "=> " LDAP_XSTRING(bdb_tool_entry_delete) ": %s\n",
1126                                 text->bv_val, 0, 0 );
1127                 goto done;
1128         }
1129
1130 done:
1131         /* Free the EntryInfo and the Entry */
1132         if( e != NULL ) {
1133                 bdb_entry_release( &op, e, 0 );
1134         }
1135
1136         if( rc == 0 ) {
1137                 if (! (slapMode & SLAP_TOOL_QUICK)) {
1138                 rc = TXN_COMMIT( tid, 0 );
1139                 if( rc != 0 ) {
1140                         snprintf( text->bv_val, text->bv_len,
1141                                         "txn_commit failed: %s (%d)",
1142                                         db_strerror(rc), rc );
1143                         Debug( LDAP_DEBUG_ANY,
1144                                 "=> " LDAP_XSTRING(bdb_tool_entry_delete) ": "
1145                                 "%s\n", text->bv_val, 0, 0 );
1146                 }
1147                 }
1148
1149         } else {
1150                 if (! (slapMode & SLAP_TOOL_QUICK)) {
1151                 TXN_ABORT( tid );
1152                 snprintf( text->bv_val, text->bv_len,
1153                         "txn_aborted! %s (%d)",
1154                         db_strerror(rc), rc );
1155                 Debug( LDAP_DEBUG_ANY,
1156                         "=> " LDAP_XSTRING(bdb_tool_entry_delete) ": %s\n",
1157                         text->bv_val, 0, 0 );
1158                 }
1159         }
1160
1161         if ( cursor == NULL )
1162         {
1163                 rc = bdb->bi_id2entry->bdi_db->cursor(
1164                         bdb->bi_id2entry->bdi_db, bdb->bi_cache.c_txn, &cursor,
1165                         bdb->bi_db_opflags );
1166         }
1167
1168         return rc;
1169 }
1170
1171 #ifdef BDB_TOOL_IDL_CACHING
1172 static int
1173 bdb_tool_idl_cmp( const void *v1, const void *v2 )
1174 {
1175         const bdb_tool_idl_cache *c1 = v1, *c2 = v2;
1176         int rc;
1177
1178         if (( rc = c1->kstr.bv_len - c2->kstr.bv_len )) return rc;
1179         return memcmp( c1->kstr.bv_val, c2->kstr.bv_val, c1->kstr.bv_len );
1180 }
1181
1182 static int
1183 bdb_tool_idl_flush_one( void *v1, void *arg )
1184 {
1185         bdb_tool_idl_cache *ic = v1;
1186         DB *db = arg;
1187         struct bdb_info *bdb = bdb_tool_info;
1188         bdb_tool_idl_cache_entry *ice;
1189         DBC *curs;
1190         DBT key, data;
1191         int i, rc;
1192         ID id, nid;
1193
1194         /* Freshly allocated, ignore it */
1195         if ( !ic->head && ic->count <= BDB_IDL_DB_SIZE ) {
1196                 return 0;
1197         }
1198
1199         rc = db->cursor( db, NULL, &curs, 0 );
1200         if ( rc )
1201                 return -1;
1202
1203         DBTzero( &key );
1204         DBTzero( &data );
1205
1206         bv2DBT( &ic->kstr, &key );
1207
1208         data.size = data.ulen = sizeof( ID );
1209         data.flags = DB_DBT_USERMEM;
1210         data.data = &nid;
1211
1212         rc = curs->c_get( curs, &key, &data, DB_SET );
1213         /* If key already exists and we're writing a range... */
1214         if ( rc == 0 && ic->count > BDB_IDL_DB_SIZE ) {
1215                 /* If it's not currently a range, must delete old info */
1216                 if ( nid ) {
1217                         /* Skip lo */
1218                         while ( curs->c_get( curs, &key, &data, DB_NEXT_DUP ) == 0 )
1219                                 curs->c_del( curs, 0 );
1220
1221                         nid = 0;
1222                         /* Store range marker */
1223                         curs->c_put( curs, &key, &data, DB_KEYFIRST );
1224                 } else {
1225
1226                         /* Skip lo */
1227                         rc = curs->c_get( curs, &key, &data, DB_NEXT_DUP );
1228
1229                         /* Get hi */
1230                         rc = curs->c_get( curs, &key, &data, DB_NEXT_DUP );
1231
1232                         /* Delete hi */
1233                         curs->c_del( curs, 0 );
1234                 }
1235                 BDB_ID2DISK( ic->last, &nid );
1236                 curs->c_put( curs, &key, &data, DB_KEYLAST );
1237                 rc = 0;
1238         } else if ( rc && rc != DB_NOTFOUND ) {
1239                 rc = -1;
1240         } else if ( ic->count > BDB_IDL_DB_SIZE ) {
1241                 /* range, didn't exist before */
1242                 nid = 0;
1243                 rc = curs->c_put( curs, &key, &data, DB_KEYLAST );
1244                 if ( rc == 0 ) {
1245                         BDB_ID2DISK( ic->first, &nid );
1246                         rc = curs->c_put( curs, &key, &data, DB_KEYLAST );
1247                         if ( rc == 0 ) {
1248                                 BDB_ID2DISK( ic->last, &nid );
1249                                 rc = curs->c_put( curs, &key, &data, DB_KEYLAST );
1250                         }
1251                 }
1252                 if ( rc ) {
1253                         rc = -1;
1254                 }
1255         } else {
1256                 int n;
1257
1258                 /* Just a normal write */
1259                 rc = 0;
1260                 for ( ice = ic->head, n=0; ice; ice = ice->next, n++ ) {
1261                         int end;
1262                         if ( ice->next ) {
1263                                 end = IDBLOCK;
1264                         } else {
1265                                 end = ic->count & (IDBLOCK-1);
1266                                 if ( !end )
1267                                         end = IDBLOCK;
1268                         }
1269                         for ( i=0; i<end; i++ ) {
1270                                 if ( !ice->ids[i] ) continue;
1271                                 BDB_ID2DISK( ice->ids[i], &nid );
1272                                 rc = curs->c_put( curs, &key, &data, DB_NODUPDATA );
1273                                 if ( rc ) {
1274                                         if ( rc == DB_KEYEXIST ) {
1275                                                 rc = 0;
1276                                                 continue;
1277                                         }
1278                                         rc = -1;
1279                                         break;
1280                                 }
1281                         }
1282                         if ( rc ) {
1283                                 rc = -1;
1284                                 break;
1285                         }
1286                 }
1287                 if ( ic->head ) {
1288                         ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
1289                         ic->tail->next = bdb_tool_idl_free_list;
1290                         bdb_tool_idl_free_list = ic->head;
1291                         bdb->bi_idl_cache_size -= n;
1292                         ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
1293                 }
1294         }
1295         if ( ic != db->app_private ) {
1296                 ch_free( ic );
1297         } else {
1298                 ic->head = ic->tail = NULL;
1299         }
1300         curs->c_close( curs );
1301         return rc;
1302 }
1303
1304 static int
1305 bdb_tool_idl_flush_db( DB *db, bdb_tool_idl_cache *ic )
1306 {
1307         Avlnode *root = db->app_private;
1308         int rc;
1309
1310         db->app_private = ic;
1311         rc = avl_apply( root, bdb_tool_idl_flush_one, db, -1, AVL_INORDER );
1312         avl_free( root, NULL );
1313         db->app_private = NULL;
1314         if ( rc != -1 )
1315                 rc = 0;
1316         return rc;
1317 }
1318
1319 static int
1320 bdb_tool_idl_flush( BackendDB *be )
1321 {
1322         struct bdb_info *bdb = (struct bdb_info *) be->be_private;
1323         DB *db;
1324         Avlnode *root;
1325         int i, rc = 0;
1326
1327         for ( i=BDB_NDB; i < bdb->bi_ndatabases; i++ ) {
1328                 db = bdb->bi_databases[i]->bdi_db;
1329                 if ( !db->app_private ) continue;
1330                 rc = bdb_tool_idl_flush_db( db, NULL );
1331                 if ( rc )
1332                         break;
1333         }
1334         if ( !rc ) {
1335                 bdb->bi_idl_cache_size = 0;
1336         }
1337         return rc;
1338 }
1339
1340 int bdb_tool_idl_add(
1341         BackendDB *be,
1342         DB *db,
1343         DB_TXN *txn,
1344         DBT *key,
1345         ID id )
1346 {
1347         struct bdb_info *bdb = (struct bdb_info *) be->be_private;
1348         bdb_tool_idl_cache *ic, itmp;
1349         bdb_tool_idl_cache_entry *ice;
1350         int rc;
1351
1352         if ( !bdb->bi_idl_cache_max_size )
1353                 return bdb_idl_insert_key( be, db, txn, key, id );
1354
1355         DBT2bv( key, &itmp.kstr );
1356
1357         ic = avl_find( (Avlnode *)db->app_private, &itmp, bdb_tool_idl_cmp );
1358
1359         /* No entry yet, create one */
1360         if ( !ic ) {
1361                 DBC *curs;
1362                 DBT data;
1363                 ID nid;
1364                 int rc;
1365
1366                 ic = ch_malloc( sizeof( bdb_tool_idl_cache ) + itmp.kstr.bv_len );
1367                 ic->kstr.bv_len = itmp.kstr.bv_len;
1368                 ic->kstr.bv_val = (char *)(ic+1);
1369                 AC_MEMCPY( ic->kstr.bv_val, itmp.kstr.bv_val, ic->kstr.bv_len );
1370                 ic->head = ic->tail = NULL;
1371                 ic->last = 0;
1372                 ic->count = 0;
1373                 avl_insert( (Avlnode **)&db->app_private, ic, bdb_tool_idl_cmp,
1374                         avl_dup_error );
1375
1376                 /* load existing key count here */
1377                 rc = db->cursor( db, NULL, &curs, 0 );
1378                 if ( rc ) return rc;
1379
1380                 data.ulen = sizeof( ID );
1381                 data.flags = DB_DBT_USERMEM;
1382                 data.data = &nid;
1383                 rc = curs->c_get( curs, key, &data, DB_SET );
1384                 if ( rc == 0 ) {
1385                         if ( nid == 0 ) {
1386                                 ic->count = BDB_IDL_DB_SIZE+1;
1387                         } else {
1388                                 db_recno_t count;
1389
1390                                 curs->c_count( curs, &count, 0 );
1391                                 ic->count = count;
1392                                 BDB_DISK2ID( &nid, &ic->first );
1393                         }
1394                 }
1395                 curs->c_close( curs );
1396         }
1397         /* are we a range already? */
1398         if ( ic->count > BDB_IDL_DB_SIZE ) {
1399                 ic->last = id;
1400                 return 0;
1401         /* Are we at the limit, and converting to a range? */
1402         } else if ( ic->count == BDB_IDL_DB_SIZE ) {
1403                 int n;
1404                 for ( ice = ic->head, n=0; ice; ice = ice->next, n++ )
1405                         /* counting */ ;
1406                 if ( n ) {
1407                         ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
1408                         ic->tail->next = bdb_tool_idl_free_list;
1409                         bdb_tool_idl_free_list = ic->head;
1410                         bdb->bi_idl_cache_size -= n;
1411                         ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
1412                 }
1413                 ic->head = ic->tail = NULL;
1414                 ic->last = id;
1415                 ic->count++;
1416                 return 0;
1417         }
1418         /* No free block, create that too */
1419         if ( !ic->tail || ( ic->count & (IDBLOCK-1)) == 0) {
1420                 ice = NULL;
1421                 ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
1422                 if ( bdb->bi_idl_cache_size >= bdb->bi_idl_cache_max_size ) {
1423                         ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
1424                         rc = bdb_tool_idl_flush_db( db, ic );
1425                         if ( rc )
1426                                 return rc;
1427                         avl_insert( (Avlnode **)&db->app_private, ic, bdb_tool_idl_cmp,
1428                                 avl_dup_error );
1429                         ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
1430                 }
1431                 bdb->bi_idl_cache_size++;
1432                 if ( bdb_tool_idl_free_list ) {
1433                         ice = bdb_tool_idl_free_list;
1434                         bdb_tool_idl_free_list = ice->next;
1435                 }
1436                 ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
1437                 if ( !ice ) {
1438                         ice = ch_malloc( sizeof( bdb_tool_idl_cache_entry ));
1439                 }
1440                 memset( ice, 0, sizeof( *ice ));
1441                 if ( !ic->head ) {
1442                         ic->head = ice;
1443                 } else {
1444                         ic->tail->next = ice;
1445                 }
1446                 ic->tail = ice;
1447                 if ( !ic->count )
1448                         ic->first = id;
1449         }
1450         ice = ic->tail;
1451         ice->ids[ ic->count & (IDBLOCK-1) ] = id;
1452         ic->count++;
1453
1454         return 0;
1455 }
1456 #endif
1457
1458 #ifdef USE_TRICKLE
1459 static void *
1460 bdb_tool_trickle_task( void *ctx, void *ptr )
1461 {
1462         DB_ENV *env = ptr;
1463         int wrote;
1464
1465         ldap_pvt_thread_mutex_lock( &bdb_tool_trickle_mutex );
1466         bdb_tool_trickle_active = 1;
1467         ldap_pvt_thread_cond_signal( &bdb_tool_trickle_cond_end );
1468         while ( 1 ) {
1469                 ldap_pvt_thread_cond_wait( &bdb_tool_trickle_cond,
1470                         &bdb_tool_trickle_mutex );
1471                 if ( slapd_shutdown )
1472                         break;
1473                 env->memp_trickle( env, 30, &wrote );
1474         }
1475         bdb_tool_trickle_active = 0;
1476         ldap_pvt_thread_cond_signal( &bdb_tool_trickle_cond_end );
1477         ldap_pvt_thread_mutex_unlock( &bdb_tool_trickle_mutex );
1478
1479         return NULL;
1480 }
1481 #endif
1482
1483 static void *
1484 bdb_tool_index_task( void *ctx, void *ptr )
1485 {
1486         int base = *(int *)ptr;
1487
1488         free( ptr );
1489         while ( 1 ) {
1490                 ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex );
1491                 bdb_tool_index_tcount--;
1492                 if ( !bdb_tool_index_tcount )
1493                         ldap_pvt_thread_cond_signal( &bdb_tool_index_cond_main );
1494                 ldap_pvt_thread_cond_wait( &bdb_tool_index_cond_work,
1495                         &bdb_tool_index_mutex );
1496                 if ( slapd_shutdown ) {
1497                         bdb_tool_index_tcount--;
1498                         if ( !bdb_tool_index_tcount )
1499                                 ldap_pvt_thread_cond_signal( &bdb_tool_index_cond_main );
1500                         ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex );
1501                         break;
1502                 }
1503                 ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex );
1504
1505                 bdb_tool_index_threads[base] = bdb_index_recrun( bdb_tool_ix_op,
1506                         bdb_tool_info, bdb_tool_index_rec, bdb_tool_ix_id, base );
1507         }
1508
1509         return NULL;
1510 }