git.sur5r.net Git - openldap/blob - servers/slapd/back-bdb/tools.c

   1 /* tools.c - tools for slap tools */
   2 /* $OpenLDAP$ */
   3 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
   4  *
   5  * Copyright 2000-2014 The OpenLDAP Foundation.
   6  * All rights reserved.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted only as authorized by the OpenLDAP
  10  * Public License.
  11  *
  12  * A copy of this license is available in the file LICENSE in the
  13  * top-level directory of the distribution or, alternatively, at
  14  * <http://www.OpenLDAP.org/license.html>.
  15  */
  16
  17 #include "portable.h"
  18
  19 #include <stdio.h>
  20 #include <ac/string.h>
  21 #include <ac/errno.h>
  22
  23 #define AVL_INTERNAL
  24 #include "back-bdb.h"
  25 #include "idl.h"
  26
  27 static DBC *cursor = NULL;
  28 static DBT key, data;
  29 static EntryHeader eh;
  30 static ID nid, previd = NOID;
  31 static char ehbuf[16];
  32
  33 typedef struct dn_id {
  34         ID id;
  35         struct berval dn;
  36 } dn_id;
  37
  38 #define HOLE_SIZE       4096
  39 static dn_id hbuf[HOLE_SIZE], *holes = hbuf;
  40 static unsigned nhmax = HOLE_SIZE;
  41 static unsigned nholes;
  42
  43 static int index_nattrs;
  44
  45 static struct berval    *tool_base;
  46 static int              tool_scope;
  47 static Filter           *tool_filter;
  48 static Entry            *tool_next_entry;
  49
  50 #ifdef BDB_TOOL_IDL_CACHING
  51 #define bdb_tool_idl_cmp                BDB_SYMBOL(tool_idl_cmp)
  52 #define bdb_tool_idl_flush_one          BDB_SYMBOL(tool_idl_flush_one)
  53 #define bdb_tool_idl_flush              BDB_SYMBOL(tool_idl_flush)
  54
  55 static int bdb_tool_idl_flush( BackendDB *be );
  56
  57 #define IDBLOCK 1024
  58
  59 typedef struct bdb_tool_idl_cache_entry {
  60         struct bdb_tool_idl_cache_entry *next;
  61         ID ids[IDBLOCK];
  62 } bdb_tool_idl_cache_entry;
  63
  64 typedef struct bdb_tool_idl_cache {
  65         struct berval kstr;
  66         bdb_tool_idl_cache_entry *head, *tail;
  67         ID first, last;
  68         int count;
  69 } bdb_tool_idl_cache;
  70
  71 static bdb_tool_idl_cache_entry *bdb_tool_idl_free_list;
  72 #endif  /* BDB_TOOL_IDL_CACHING */
  73
  74 static ID bdb_tool_ix_id;
  75 static Operation *bdb_tool_ix_op;
  76 static int *bdb_tool_index_threads, bdb_tool_index_tcount;
  77 static void *bdb_tool_index_rec;
  78 static struct bdb_info *bdb_tool_info;
  79 static ldap_pvt_thread_mutex_t bdb_tool_index_mutex;
  80 static ldap_pvt_thread_cond_t bdb_tool_index_cond_main;
  81 static ldap_pvt_thread_cond_t bdb_tool_index_cond_work;
  82
  83 #if DB_VERSION_FULL >= 0x04060000
  84 #define USE_TRICKLE     1
  85 #else
  86 /* Seems to slow things down too much in BDB 4.5 */
  87 #undef USE_TRICKLE
  88 #endif
  89
  90 #ifdef USE_TRICKLE
  91 static ldap_pvt_thread_mutex_t bdb_tool_trickle_mutex;
  92 static ldap_pvt_thread_cond_t bdb_tool_trickle_cond;
  93 static ldap_pvt_thread_cond_t bdb_tool_trickle_cond_end;
  94
  95 static void * bdb_tool_trickle_task( void *ctx, void *ptr );
  96 static int bdb_tool_trickle_active;
  97 #endif
  98
  99 static void * bdb_tool_index_task( void *ctx, void *ptr );
 100
 101 static int
 102 bdb_tool_entry_get_int( BackendDB *be, ID id, Entry **ep );
 103
 104 static int bdb_tool_threads;
 105
 106 int bdb_tool_entry_open(
 107         BackendDB *be, int mode )
 108 {
 109         struct bdb_info *bdb = (struct bdb_info *) be->be_private;
 110
 111         /* initialize key and data thangs */
 112         DBTzero( &key );
 113         DBTzero( &data );
 114         key.flags = DB_DBT_USERMEM;
 115         key.data = &nid;
 116         key.size = key.ulen = sizeof( nid );
 117         data.flags = DB_DBT_USERMEM;
 118
 119         if (cursor == NULL) {
 120                 int rc = bdb->bi_id2entry->bdi_db->cursor(
 121                         bdb->bi_id2entry->bdi_db, bdb->bi_cache.c_txn, &cursor,
 122                         bdb->bi_db_opflags );
 123                 if( rc != 0 ) {
 124                         return -1;
 125                 }
 126         }
 127
 128         /* Set up for threaded slapindex */
 129         if (( slapMode & (SLAP_TOOL_QUICK|SLAP_TOOL_READONLY)) == SLAP_TOOL_QUICK ) {
 130                 if ( !bdb_tool_info ) {
 131 #ifdef USE_TRICKLE
 132                         ldap_pvt_thread_mutex_init( &bdb_tool_trickle_mutex );
 133                         ldap_pvt_thread_cond_init( &bdb_tool_trickle_cond );
 134                         ldap_pvt_thread_cond_init( &bdb_tool_trickle_cond_end );
 135                         ldap_pvt_thread_pool_submit( &connection_pool, bdb_tool_trickle_task, bdb->bi_dbenv );
 136 #endif
 137
 138                         ldap_pvt_thread_mutex_init( &bdb_tool_index_mutex );
 139                         ldap_pvt_thread_cond_init( &bdb_tool_index_cond_main );
 140                         ldap_pvt_thread_cond_init( &bdb_tool_index_cond_work );
 141                         if ( bdb->bi_nattrs ) {
 142                                 int i;
 143                                 bdb_tool_threads = slap_tool_thread_max - 1;
 144                                 if ( bdb_tool_threads > 1 ) {
 145                                         bdb_tool_index_threads = ch_malloc( bdb_tool_threads * sizeof( int ));
 146                                         bdb_tool_index_rec = ch_malloc( bdb->bi_nattrs * sizeof( IndexRec ));
 147                                         bdb_tool_index_tcount = bdb_tool_threads - 1;
 148                                         for (i=1; i<bdb_tool_threads; i++) {
 149                                                 int *ptr = ch_malloc( sizeof( int ));
 150                                                 *ptr = i;
 151                                                 ldap_pvt_thread_pool_submit( &connection_pool,
 152                                                         bdb_tool_index_task, ptr );
 153                                         }
 154                                 }
 155                         }
 156                         bdb_tool_info = bdb;
 157                 }
 158         }
 159
 160         return 0;
 161 }
 162
 163 int bdb_tool_entry_close(
 164         BackendDB *be )
 165 {
 166         if ( bdb_tool_info ) {
 167                 slapd_shutdown = 1;
 168 #ifdef USE_TRICKLE
 169                 ldap_pvt_thread_mutex_lock( &bdb_tool_trickle_mutex );
 170
 171                 /* trickle thread may not have started yet */
 172                 while ( !bdb_tool_trickle_active )
 173                         ldap_pvt_thread_cond_wait( &bdb_tool_trickle_cond_end,
 174                                         &bdb_tool_trickle_mutex );
 175
 176                 ldap_pvt_thread_cond_signal( &bdb_tool_trickle_cond );
 177                 while ( bdb_tool_trickle_active )
 178                         ldap_pvt_thread_cond_wait( &bdb_tool_trickle_cond_end,
 179                                         &bdb_tool_trickle_mutex );
 180                 ldap_pvt_thread_mutex_unlock( &bdb_tool_trickle_mutex );
 181 #endif
 182                 if ( bdb_tool_threads > 1 ) {
 183                         ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex );
 184
 185                         /* There might still be some threads starting */
 186                         while ( bdb_tool_index_tcount > 0 ) {
 187                                 ldap_pvt_thread_cond_wait( &bdb_tool_index_cond_main,
 188                                                 &bdb_tool_index_mutex );
 189                         }
 190
 191                         bdb_tool_index_tcount = bdb_tool_threads - 1;
 192                         ldap_pvt_thread_cond_broadcast( &bdb_tool_index_cond_work );
 193
 194                         /* Make sure all threads are stopped */
 195                         while ( bdb_tool_index_tcount > 0 ) {
 196                                 ldap_pvt_thread_cond_wait( &bdb_tool_index_cond_main,
 197                                         &bdb_tool_index_mutex );
 198                         }
 199                         ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex );
 200
 201                         ch_free( bdb_tool_index_threads );
 202                         ch_free( bdb_tool_index_rec );
 203                         bdb_tool_index_tcount = bdb_tool_threads - 1;
 204                 }
 205                 bdb_tool_info = NULL;
 206                 slapd_shutdown = 0;
 207         }
 208
 209         if( eh.bv.bv_val ) {
 210                 ch_free( eh.bv.bv_val );
 211                 eh.bv.bv_val = NULL;
 212         }
 213
 214         if( cursor ) {
 215                 cursor->c_close( cursor );
 216                 cursor = NULL;
 217         }
 218
 219 #ifdef BDB_TOOL_IDL_CACHING
 220         bdb_tool_idl_flush( be );
 221 #endif
 222
 223         if( nholes ) {
 224                 unsigned i;
 225                 fprintf( stderr, "Error, entries missing!\n");
 226                 for (i=0; i<nholes; i++) {
 227                         fprintf(stderr, "  entry %ld: %s\n",
 228                                 holes[i].id, holes[i].dn.bv_val);
 229                 }
 230                 return -1;
 231         }
 232
 233         return 0;
 234 }
 235
 236 ID
 237 bdb_tool_entry_first_x(
 238         BackendDB *be,
 239         struct berval *base,
 240         int scope,
 241         Filter *f )
 242 {
 243         tool_base = base;
 244         tool_scope = scope;
 245         tool_filter = f;
 246
 247         return bdb_tool_entry_next( be );
 248 }
 249
 250 ID bdb_tool_entry_next(
 251         BackendDB *be )
 252 {
 253         int rc;
 254         ID id;
 255         struct bdb_info *bdb;
 256
 257         assert( be != NULL );
 258         assert( slapMode & SLAP_TOOL_MODE );
 259
 260         bdb = (struct bdb_info *) be->be_private;
 261         assert( bdb != NULL );
 262
 263 next:;
 264         /* Get the header */
 265         data.ulen = data.dlen = sizeof( ehbuf );
 266         data.data = ehbuf;
 267         data.flags |= DB_DBT_PARTIAL;
 268         rc = cursor->c_get( cursor, &key, &data, DB_NEXT );
 269
 270         if( rc ) {
 271                 /* If we're doing linear indexing and there are more attrs to
 272                  * index, and we're at the end of the database, start over.
 273                  */
 274                 if ( index_nattrs && rc == DB_NOTFOUND ) {
 275                         /* optional - do a checkpoint here? */
 276                         bdb_attr_info_free( bdb->bi_attrs[0] );
 277                         bdb->bi_attrs[0] = bdb->bi_attrs[index_nattrs];
 278                         index_nattrs--;
 279                         rc = cursor->c_get( cursor, &key, &data, DB_FIRST );
 280                         if ( rc ) {
 281                                 return NOID;
 282                         }
 283                 } else {
 284                         return NOID;
 285                 }
 286         }
 287
 288         BDB_DISK2ID( key.data, &id );
 289         previd = id;
 290
 291         if ( tool_filter || tool_base ) {
 292                 static Operation op = {0};
 293                 static Opheader ohdr = {0};
 294
 295                 op.o_hdr = &ohdr;
 296                 op.o_bd = be;
 297                 op.o_tmpmemctx = NULL;
 298                 op.o_tmpmfuncs = &ch_mfuncs;
 299
 300                 if ( tool_next_entry ) {
 301                         bdb_entry_release( &op, tool_next_entry, 0 );
 302                         tool_next_entry = NULL;
 303                 }
 304
 305                 rc = bdb_tool_entry_get_int( be, id, &tool_next_entry );
 306                 if ( rc == LDAP_NO_SUCH_OBJECT ) {
 307                         goto next;
 308                 }
 309
 310                 assert( tool_next_entry != NULL );
 311
 312 #ifdef BDB_HIER
 313                 /* TODO: needed until BDB_HIER is handled accordingly
 314                  * in bdb_tool_entry_get_int() */
 315                 if ( tool_base && !dnIsSuffixScope( &tool_next_entry->e_nname, tool_base, tool_scope ) )
 316                 {
 317                         bdb_entry_release( &op, tool_next_entry, 0 );
 318                         tool_next_entry = NULL;
 319                         goto next;
 320                 }
 321 #endif
 322
 323                 if ( tool_filter && test_filter( NULL, tool_next_entry, tool_filter ) != LDAP_COMPARE_TRUE )
 324                 {
 325                         bdb_entry_release( &op, tool_next_entry, 0 );
 326                         tool_next_entry = NULL;
 327                         goto next;
 328                 }
 329         }
 330
 331         return id;
 332 }
 333
 334 ID bdb_tool_dn2id_get(
 335         Backend *be,
 336         struct berval *dn
 337 )
 338 {
 339         Operation op = {0};
 340         Opheader ohdr = {0};
 341         EntryInfo *ei = NULL;
 342         int rc;
 343
 344         if ( BER_BVISEMPTY(dn) )
 345                 return 0;
 346
 347         op.o_hdr = &ohdr;
 348         op.o_bd = be;
 349         op.o_tmpmemctx = NULL;
 350         op.o_tmpmfuncs = &ch_mfuncs;
 351
 352         rc = bdb_cache_find_ndn( &op, 0, dn, &ei );
 353         if ( ei ) bdb_cache_entryinfo_unlock( ei );
 354         if ( rc == DB_NOTFOUND )
 355                 return NOID;
 356
 357         return ei->bei_id;
 358 }
 359
 360 static int
 361 bdb_tool_entry_get_int( BackendDB *be, ID id, Entry **ep )
 362 {
 363         Entry *e = NULL;
 364         char *dptr;
 365         int rc, eoff;
 366
 367         assert( be != NULL );
 368         assert( slapMode & SLAP_TOOL_MODE );
 369
 370         if ( ( tool_filter || tool_base ) && id == previd && tool_next_entry != NULL ) {
 371                 *ep = tool_next_entry;
 372                 tool_next_entry = NULL;
 373                 return LDAP_SUCCESS;
 374         }
 375
 376         if ( id != previd ) {
 377                 data.ulen = data.dlen = sizeof( ehbuf );
 378                 data.data = ehbuf;
 379                 data.flags |= DB_DBT_PARTIAL;
 380
 381                 BDB_ID2DISK( id, &nid );
 382                 rc = cursor->c_get( cursor, &key, &data, DB_SET );
 383                 if ( rc ) {
 384                         rc = LDAP_OTHER;
 385                         goto done;
 386                 }
 387         }
 388
 389         /* Get the header */
 390         dptr = eh.bv.bv_val;
 391         eh.bv.bv_val = ehbuf;
 392         eh.bv.bv_len = data.size;
 393         rc = entry_header( &eh );
 394         eoff = eh.data - eh.bv.bv_val;
 395         eh.bv.bv_val = dptr;
 396         if ( rc ) {
 397                 rc = LDAP_OTHER;
 398                 goto done;
 399         }
 400
 401         /* Get the size */
 402         data.flags &= ~DB_DBT_PARTIAL;
 403         data.ulen = 0;
 404         rc = cursor->c_get( cursor, &key, &data, DB_CURRENT );
 405         if ( rc != DB_BUFFER_SMALL ) {
 406                 rc = LDAP_OTHER;
 407                 goto done;
 408         }
 409
 410         /* Allocate a block and retrieve the data */
 411         eh.bv.bv_len = eh.nvals * sizeof( struct berval ) + data.size;
 412         eh.bv.bv_val = ch_realloc( eh.bv.bv_val, eh.bv.bv_len );
 413         eh.data = eh.bv.bv_val + eh.nvals * sizeof( struct berval );
 414         data.data = eh.data;
 415         data.ulen = data.size;
 416
 417         /* Skip past already parsed nattr/nvals */
 418         eh.data += eoff;
 419
 420         rc = cursor->c_get( cursor, &key, &data, DB_CURRENT );
 421         if ( rc ) {
 422                 rc = LDAP_OTHER;
 423                 goto done;
 424         }
 425
 426 #ifndef BDB_HIER
 427         /* TODO: handle BDB_HIER accordingly */
 428         if ( tool_base != NULL ) {
 429                 struct berval ndn;
 430                 entry_decode_dn( &eh, NULL, &ndn );
 431
 432                 if ( !dnIsSuffixScope( &ndn, tool_base, tool_scope ) ) {
 433                         return LDAP_NO_SUCH_OBJECT;
 434                 }
 435         }
 436 #endif
 437
 438 #ifdef SLAP_ZONE_ALLOC
 439         /* FIXME: will add ctx later */
 440         rc = entry_decode( &eh, &e, NULL );
 441 #else
 442         rc = entry_decode( &eh, &e );
 443 #endif
 444
 445         if( rc == LDAP_SUCCESS ) {
 446                 e->e_id = id;
 447 #ifdef BDB_HIER
 448                 if ( slapMode & SLAP_TOOL_READONLY ) {
 449                         struct bdb_info *bdb = (struct bdb_info *) be->be_private;
 450                         EntryInfo *ei = NULL;
 451                         Operation op = {0};
 452                         Opheader ohdr = {0};
 453
 454                         op.o_hdr = &ohdr;
 455                         op.o_bd = be;
 456                         op.o_tmpmemctx = NULL;
 457                         op.o_tmpmfuncs = &ch_mfuncs;
 458
 459                         rc = bdb_cache_find_parent( &op, bdb->bi_cache.c_txn, id, &ei );
 460                         if ( rc == LDAP_SUCCESS ) {
 461                                 bdb_cache_entryinfo_unlock( ei );
 462                                 e->e_private = ei;
 463                                 ei->bei_e = e;
 464                                 bdb_fix_dn( e, 0 );
 465                                 ei->bei_e = NULL;
 466                                 e->e_private = NULL;
 467                         }
 468                 }
 469 #endif
 470         }
 471 done:
 472         if ( e != NULL ) {
 473                 *ep = e;
 474         }
 475
 476         return rc;
 477 }
 478
 479 Entry*
 480 bdb_tool_entry_get( BackendDB *be, ID id )
 481 {
 482         Entry *e = NULL;
 483
 484         (void)bdb_tool_entry_get_int( be, id, &e );
 485         return e;
 486 }
 487
 488 static int bdb_tool_next_id(
 489         Operation *op,
 490         DB_TXN *tid,
 491         Entry *e,
 492         struct berval *text,
 493         int hole )
 494 {
 495         struct berval dn = e->e_name;
 496         struct berval ndn = e->e_nname;
 497         struct berval pdn, npdn;
 498         EntryInfo *ei = NULL, eidummy;
 499         int rc;
 500
 501         if (ndn.bv_len == 0) {
 502                 e->e_id = 0;
 503                 return 0;
 504         }
 505
 506         rc = bdb_cache_find_ndn( op, tid, &ndn, &ei );
 507         if ( ei ) bdb_cache_entryinfo_unlock( ei );
 508         if ( rc == DB_NOTFOUND ) {
 509                 if ( !be_issuffix( op->o_bd, &ndn ) ) {
 510                         ID eid = e->e_id;
 511                         dnParent( &dn, &pdn );
 512                         dnParent( &ndn, &npdn );
 513                         e->e_name = pdn;
 514                         e->e_nname = npdn;
 515                         rc = bdb_tool_next_id( op, tid, e, text, 1 );
 516                         e->e_name = dn;
 517                         e->e_nname = ndn;
 518                         if ( rc ) {
 519                                 return rc;
 520                         }
 521                         /* If parent didn't exist, it was created just now
 522                          * and its ID is now in e->e_id. Make sure the current
 523                          * entry gets added under the new parent ID.
 524                          */
 525                         if ( eid != e->e_id ) {
 526                                 eidummy.bei_id = e->e_id;
 527                                 ei = &eidummy;
 528                         }
 529                 }
 530                 rc = bdb_next_id( op->o_bd, &e->e_id );
 531                 if ( rc ) {
 532                         snprintf( text->bv_val, text->bv_len,
 533                                 "next_id failed: %s (%d)",
 534                                 db_strerror(rc), rc );
 535                 Debug( LDAP_DEBUG_ANY,
 536                         "=> bdb_tool_next_id: %s\n", text->bv_val, 0, 0 );
 537                         return rc;
 538                 }
 539                 rc = bdb_dn2id_add( op, tid, ei, e );
 540                 if ( rc ) {
 541                         snprintf( text->bv_val, text->bv_len,
 542                                 "dn2id_add failed: %s (%d)",
 543                                 db_strerror(rc), rc );
 544                 Debug( LDAP_DEBUG_ANY,
 545                         "=> bdb_tool_next_id: %s\n", text->bv_val, 0, 0 );
 546                 } else if ( hole ) {
 547                         if ( nholes == nhmax - 1 ) {
 548                                 if ( holes == hbuf ) {
 549                                         holes = ch_malloc( nhmax * sizeof(dn_id) * 2 );
 550                                         AC_MEMCPY( holes, hbuf, sizeof(hbuf) );
 551                                 } else {
 552                                         holes = ch_realloc( holes, nhmax * sizeof(dn_id) * 2 );
 553                                 }
 554                                 nhmax *= 2;
 555                         }
 556                         ber_dupbv( &holes[nholes].dn, &ndn );
 557                         holes[nholes++].id = e->e_id;
 558                 }
 559         } else if ( !hole ) {
 560                 unsigned i, j;
 561
 562                 e->e_id = ei->bei_id;
 563
 564                 for ( i=0; i<nholes; i++) {
 565                         if ( holes[i].id == e->e_id ) {
 566                                 free(holes[i].dn.bv_val);
 567                                 for (j=i;j<nholes;j++) holes[j] = holes[j+1];
 568                                 holes[j].id = 0;
 569                                 nholes--;
 570                                 break;
 571                         } else if ( holes[i].id > e->e_id ) {
 572                                 break;
 573                         }
 574                 }
 575         }
 576         return rc;
 577 }
 578
 579 static int
 580 bdb_tool_index_add(
 581         Operation *op,
 582         DB_TXN *txn,
 583         Entry *e )
 584 {
 585         struct bdb_info *bdb = (struct bdb_info *) op->o_bd->be_private;
 586
 587         if ( !bdb->bi_nattrs )
 588                 return 0;
 589
 590         if ( bdb_tool_threads > 1 ) {
 591                 IndexRec *ir;
 592                 int i, rc;
 593                 Attribute *a;
 594
 595                 ir = bdb_tool_index_rec;
 596                 memset(ir, 0, bdb->bi_nattrs * sizeof( IndexRec ));
 597
 598                 for ( a = e->e_attrs; a != NULL; a = a->a_next ) {
 599                         rc = bdb_index_recset( bdb, a, a->a_desc->ad_type,
 600                                 &a->a_desc->ad_tags, ir );
 601                         if ( rc )
 602                                 return rc;
 603                 }
 604                 bdb_tool_ix_id = e->e_id;
 605                 bdb_tool_ix_op = op;
 606                 ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex );
 607                 /* Wait for all threads to be ready */
 608                 while ( bdb_tool_index_tcount > 0 ) {
 609                         ldap_pvt_thread_cond_wait( &bdb_tool_index_cond_main,
 610                                 &bdb_tool_index_mutex );
 611                 }
 612                 for ( i=1; i<bdb_tool_threads; i++ )
 613                         bdb_tool_index_threads[i] = LDAP_BUSY;
 614                 bdb_tool_index_tcount = bdb_tool_threads - 1;
 615                 ldap_pvt_thread_cond_broadcast( &bdb_tool_index_cond_work );
 616                 ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex );
 617                 rc = bdb_index_recrun( op, bdb, ir, e->e_id, 0 );
 618                 if ( rc )
 619                         return rc;
 620                 ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex );
 621                 for ( i=1; i<bdb_tool_threads; i++ ) {
 622                         if ( bdb_tool_index_threads[i] == LDAP_BUSY ) {
 623                                 ldap_pvt_thread_cond_wait( &bdb_tool_index_cond_main,
 624                                         &bdb_tool_index_mutex );
 625                                 i--;
 626                                 continue;
 627                         }
 628                         if ( bdb_tool_index_threads[i] ) {
 629                                 rc = bdb_tool_index_threads[i];
 630                                 break;
 631                         }
 632                 }
 633                 ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex );
 634                 return rc;
 635         } else {
 636                 return bdb_index_entry_add( op, txn, e );
 637         }
 638 }
 639
 640 ID bdb_tool_entry_put(
 641         BackendDB *be,
 642         Entry *e,
 643         struct berval *text )
 644 {
 645         int rc;
 646         struct bdb_info *bdb;
 647         DB_TXN *tid = NULL;
 648         Operation op = {0};
 649         Opheader ohdr = {0};
 650
 651         assert( be != NULL );
 652         assert( slapMode & SLAP_TOOL_MODE );
 653
 654         assert( text != NULL );
 655         assert( text->bv_val != NULL );
 656         assert( text->bv_val[0] == '\0' );      /* overconservative? */
 657
 658         Debug( LDAP_DEBUG_TRACE, "=> " LDAP_XSTRING(bdb_tool_entry_put)
 659                 "( %ld, \"%s\" )\n", (long) e->e_id, e->e_dn, 0 );
 660
 661         bdb = (struct bdb_info *) be->be_private;
 662
 663         if (! (slapMode & SLAP_TOOL_QUICK)) {
 664         rc = TXN_BEGIN( bdb->bi_dbenv, NULL, &tid,
 665                 bdb->bi_db_opflags );
 666         if( rc != 0 ) {
 667                 snprintf( text->bv_val, text->bv_len,
 668                         "txn_begin failed: %s (%d)",
 669                         db_strerror(rc), rc );
 670                 Debug( LDAP_DEBUG_ANY,
 671                         "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
 672                          text->bv_val, 0, 0 );
 673                 return NOID;
 674         }
 675         Debug( LDAP_DEBUG_TRACE, LDAP_XSTRING(bdb_tool_entry_put) ": txn id: %x\n",
 676                 tid->id(tid), 0, 0 );
 677         }
 678
 679         op.o_hdr = &ohdr;
 680         op.o_bd = be;
 681         op.o_tmpmemctx = NULL;
 682         op.o_tmpmfuncs = &ch_mfuncs;
 683
 684         /* add dn2id indices */
 685         rc = bdb_tool_next_id( &op, tid, e, text, 0 );
 686         if( rc != 0 ) {
 687                 goto done;
 688         }
 689
 690 #ifdef USE_TRICKLE
 691         if (( slapMode & SLAP_TOOL_QUICK ) && (( e->e_id & 0xfff ) == 0xfff )) {
 692                 ldap_pvt_thread_cond_signal( &bdb_tool_trickle_cond );
 693         }
 694 #endif
 695
 696         if ( !bdb->bi_linear_index )
 697                 rc = bdb_tool_index_add( &op, tid, e );
 698         if( rc != 0 ) {
 699                 snprintf( text->bv_val, text->bv_len,
 700                                 "index_entry_add failed: %s (%d)",
 701                                 rc == LDAP_OTHER ? "Internal error" :
 702                                 db_strerror(rc), rc );
 703                 Debug( LDAP_DEBUG_ANY,
 704                         "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
 705                         text->bv_val, 0, 0 );
 706                 goto done;
 707         }
 708
 709         /* id2entry index */
 710         rc = bdb_id2entry_add( be, tid, e );
 711         if( rc != 0 ) {
 712                 snprintf( text->bv_val, text->bv_len,
 713                                 "id2entry_add failed: %s (%d)",
 714                                 db_strerror(rc), rc );
 715                 Debug( LDAP_DEBUG_ANY,
 716                         "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
 717                         text->bv_val, 0, 0 );
 718                 goto done;
 719         }
 720
 721 done:
 722         if( rc == 0 ) {
 723                 if ( !( slapMode & SLAP_TOOL_QUICK )) {
 724                 rc = TXN_COMMIT( tid, 0 );
 725                 if( rc != 0 ) {
 726                         snprintf( text->bv_val, text->bv_len,
 727                                         "txn_commit failed: %s (%d)",
 728                                         db_strerror(rc), rc );
 729                         Debug( LDAP_DEBUG_ANY,
 730                                 "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
 731                                 text->bv_val, 0, 0 );
 732                         e->e_id = NOID;
 733                 }
 734                 }
 735
 736         } else {
 737                 if ( !( slapMode & SLAP_TOOL_QUICK )) {
 738                 TXN_ABORT( tid );
 739                 snprintf( text->bv_val, text->bv_len,
 740                         "txn_aborted! %s (%d)",
 741                         rc == LDAP_OTHER ? "Internal error" :
 742                         db_strerror(rc), rc );
 743                 Debug( LDAP_DEBUG_ANY,
 744                         "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
 745                         text->bv_val, 0, 0 );
 746                 }
 747                 e->e_id = NOID;
 748         }
 749
 750         if ( cursor == NULL )
 751         {
 752                 int rc = bdb->bi_id2entry->bdi_db->cursor(
 753                         bdb->bi_id2entry->bdi_db, bdb->bi_cache.c_txn, &cursor,
 754                         bdb->bi_db_opflags );
 755                 if ( rc != 0 )
 756                         e->e_id = NOID;
 757         }
 758
 759         return e->e_id;
 760 }
 761
 762 int bdb_tool_entry_reindex(
 763         BackendDB *be,
 764         ID id,
 765         AttributeDescription **adv )
 766 {
 767         struct bdb_info *bi = (struct bdb_info *) be->be_private;
 768         int rc;
 769         Entry *e;
 770         DB_TXN *tid = NULL;
 771         Operation op = {0};
 772         Opheader ohdr = {0};
 773
 774         Debug( LDAP_DEBUG_ARGS,
 775                 "=> " LDAP_XSTRING(bdb_tool_entry_reindex) "( %ld )\n",
 776                 (long) id, 0, 0 );
 777         assert( tool_base == NULL );
 778         assert( tool_filter == NULL );
 779
 780         /* No indexes configured, nothing to do. Could return an
 781          * error here to shortcut things.
 782          */
 783         if (!bi->bi_attrs) {
 784                 return 0;
 785         }
 786
 787         /* Check for explicit list of attrs to index */
 788         if ( adv ) {
 789                 int i, j, n;
 790
 791                 if ( bi->bi_attrs[0]->ai_desc != adv[0] ) {
 792                         /* count */
 793                         for ( n = 0; adv[n]; n++ ) ;
 794
 795                         /* insertion sort */
 796                         for ( i = 0; i < n; i++ ) {
 797                                 AttributeDescription *ad = adv[i];
 798                                 for ( j = i-1; j>=0; j--) {
 799                                         if ( SLAP_PTRCMP( adv[j], ad ) <= 0 ) break;
 800                                         adv[j+1] = adv[j];
 801                                 }
 802                                 adv[j+1] = ad;
 803                         }
 804                 }
 805
 806                 for ( i = 0; adv[i]; i++ ) {
 807                         if ( bi->bi_attrs[i]->ai_desc != adv[i] ) {
 808                                 for ( j = i+1; j < bi->bi_nattrs; j++ ) {
 809                                         if ( bi->bi_attrs[j]->ai_desc == adv[i] ) {
 810                                                 AttrInfo *ai = bi->bi_attrs[i];
 811                                                 bi->bi_attrs[i] = bi->bi_attrs[j];
 812                                                 bi->bi_attrs[j] = ai;
 813                                                 break;
 814                                         }
 815                                 }
 816                                 if ( j == bi->bi_nattrs ) {
 817                                         Debug( LDAP_DEBUG_ANY,
 818                                                 LDAP_XSTRING(bdb_tool_entry_reindex)
 819                                                 ": no index configured for %s\n",
 820                                                 adv[i]->ad_cname.bv_val, 0, 0 );
 821                                         return -1;
 822                                 }
 823                         }
 824                 }
 825                 bi->bi_nattrs = i;
 826         }
 827
 828         /* Get the first attribute to index */
 829         if (bi->bi_linear_index && !index_nattrs) {
 830                 index_nattrs = bi->bi_nattrs - 1;
 831                 bi->bi_nattrs = 1;
 832         }
 833
 834         e = bdb_tool_entry_get( be, id );
 835
 836         if( e == NULL ) {
 837                 Debug( LDAP_DEBUG_ANY,
 838                         LDAP_XSTRING(bdb_tool_entry_reindex)
 839                         ": could not locate id=%ld\n",
 840                         (long) id, 0, 0 );
 841                 return -1;
 842         }
 843
 844         if (! (slapMode & SLAP_TOOL_QUICK)) {
 845         rc = TXN_BEGIN( bi->bi_dbenv, NULL, &tid, bi->bi_db_opflags );
 846         if( rc != 0 ) {
 847                 Debug( LDAP_DEBUG_ANY,
 848                         "=> " LDAP_XSTRING(bdb_tool_entry_reindex) ": "
 849                         "txn_begin failed: %s (%d)\n",
 850                         db_strerror(rc), rc, 0 );
 851                 goto done;
 852         }
 853         Debug( LDAP_DEBUG_TRACE, LDAP_XSTRING(bdb_tool_entry_reindex) ": txn id: %x\n",
 854                 tid->id(tid), 0, 0 );
 855         }
 856
 857         /*
 858          * just (re)add them for now
 859          * assume that some other routine (not yet implemented)
 860          * will zap index databases
 861          *
 862          */
 863
 864         Debug( LDAP_DEBUG_TRACE,
 865                 "=> " LDAP_XSTRING(bdb_tool_entry_reindex) "( %ld, \"%s\" )\n",
 866                 (long) id, e->e_dn, 0 );
 867
 868         op.o_hdr = &ohdr;
 869         op.o_bd = be;
 870         op.o_tmpmemctx = NULL;
 871         op.o_tmpmfuncs = &ch_mfuncs;
 872
 873         rc = bdb_tool_index_add( &op, tid, e );
 874
 875 done:
 876         if( rc == 0 ) {
 877                 if (! (slapMode & SLAP_TOOL_QUICK)) {
 878                 rc = TXN_COMMIT( tid, 0 );
 879                 if( rc != 0 ) {
 880                         Debug( LDAP_DEBUG_ANY,
 881                                 "=> " LDAP_XSTRING(bdb_tool_entry_reindex)
 882                                 ": txn_commit failed: %s (%d)\n",
 883                                 db_strerror(rc), rc, 0 );
 884                         e->e_id = NOID;
 885                 }
 886                 }
 887
 888         } else {
 889                 if (! (slapMode & SLAP_TOOL_QUICK)) {
 890                 TXN_ABORT( tid );
 891                 Debug( LDAP_DEBUG_ANY,
 892                         "=> " LDAP_XSTRING(bdb_tool_entry_reindex)
 893                         ": txn_aborted! %s (%d)\n",
 894                         db_strerror(rc), rc, 0 );
 895                 }
 896                 e->e_id = NOID;
 897         }
 898         bdb_entry_release( &op, e, 0 );
 899
 900         return rc;
 901 }
 902
 903 ID bdb_tool_entry_modify(
 904         BackendDB *be,
 905         Entry *e,
 906         struct berval *text )
 907 {
 908         int rc;
 909         struct bdb_info *bdb;
 910         DB_TXN *tid = NULL;
 911         Operation op = {0};
 912         Opheader ohdr = {0};
 913
 914         assert( be != NULL );
 915         assert( slapMode & SLAP_TOOL_MODE );
 916
 917         assert( text != NULL );
 918         assert( text->bv_val != NULL );
 919         assert( text->bv_val[0] == '\0' );      /* overconservative? */
 920
 921         assert ( e->e_id != NOID );
 922
 923         Debug( LDAP_DEBUG_TRACE,
 924                 "=> " LDAP_XSTRING(bdb_tool_entry_modify) "( %ld, \"%s\" )\n",
 925                 (long) e->e_id, e->e_dn, 0 );
 926
 927         bdb = (struct bdb_info *) be->be_private;
 928
 929         if (! (slapMode & SLAP_TOOL_QUICK)) {
 930                 if( cursor ) {
 931                         cursor->c_close( cursor );
 932                         cursor = NULL;
 933                 }
 934                 rc = TXN_BEGIN( bdb->bi_dbenv, NULL, &tid,
 935                         bdb->bi_db_opflags );
 936                 if( rc != 0 ) {
 937                         snprintf( text->bv_val, text->bv_len,
 938                                 "txn_begin failed: %s (%d)",
 939                                 db_strerror(rc), rc );
 940                         Debug( LDAP_DEBUG_ANY,
 941                                 "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n",
 942                                  text->bv_val, 0, 0 );
 943                         return NOID;
 944                 }
 945                 Debug( LDAP_DEBUG_TRACE, LDAP_XSTRING(bdb_tool_entry_modify) ": txn id: %x\n",
 946                         tid->id(tid), 0, 0 );
 947         }
 948
 949         op.o_hdr = &ohdr;
 950         op.o_bd = be;
 951         op.o_tmpmemctx = NULL;
 952         op.o_tmpmfuncs = &ch_mfuncs;
 953
 954         /* id2entry index */
 955         rc = bdb_id2entry_update( be, tid, e );
 956         if( rc != 0 ) {
 957                 snprintf( text->bv_val, text->bv_len,
 958                                 "id2entry_add failed: %s (%d)",
 959                                 db_strerror(rc), rc );
 960                 Debug( LDAP_DEBUG_ANY,
 961                         "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n",
 962                         text->bv_val, 0, 0 );
 963                 goto done;
 964         }
 965
 966 done:
 967         if( rc == 0 ) {
 968                 if (! (slapMode & SLAP_TOOL_QUICK)) {
 969                 rc = TXN_COMMIT( tid, 0 );
 970                 if( rc != 0 ) {
 971                         snprintf( text->bv_val, text->bv_len,
 972                                         "txn_commit failed: %s (%d)",
 973                                         db_strerror(rc), rc );
 974                         Debug( LDAP_DEBUG_ANY,
 975                                 "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": "
 976                                 "%s\n", text->bv_val, 0, 0 );
 977                         e->e_id = NOID;
 978                 }
 979                 }
 980
 981         } else {
 982                 if (! (slapMode & SLAP_TOOL_QUICK)) {
 983                 TXN_ABORT( tid );
 984                 snprintf( text->bv_val, text->bv_len,
 985                         "txn_aborted! %s (%d)",
 986                         db_strerror(rc), rc );
 987                 Debug( LDAP_DEBUG_ANY,
 988                         "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n",
 989                         text->bv_val, 0, 0 );
 990                 }
 991                 e->e_id = NOID;
 992         }
 993
 994         if ( cursor == NULL )
 995         {
 996                 int rc = bdb->bi_id2entry->bdi_db->cursor(
 997                         bdb->bi_id2entry->bdi_db, bdb->bi_cache.c_txn, &cursor,
 998                         bdb->bi_db_opflags );
 999                 if ( rc != 0 )
1000                         e->e_id = NOID;
1001         }
1002
1003         return e->e_id;
1004 }
1005
1006 #ifdef BDB_TOOL_IDL_CACHING
1007 static int
1008 bdb_tool_idl_cmp( const void *v1, const void *v2 )
1009 {
1010         const bdb_tool_idl_cache *c1 = v1, *c2 = v2;
1011         int rc;
1012
1013         if (( rc = c1->kstr.bv_len - c2->kstr.bv_len )) return rc;
1014         return memcmp( c1->kstr.bv_val, c2->kstr.bv_val, c1->kstr.bv_len );
1015 }
1016
1017 static int
1018 bdb_tool_idl_flush_one( void *v1, void *arg )
1019 {
1020         bdb_tool_idl_cache *ic = v1;
1021         DB *db = arg;
1022         struct bdb_info *bdb = bdb_tool_info;
1023         bdb_tool_idl_cache_entry *ice;
1024         DBC *curs;
1025         DBT key, data;
1026         int i, rc;
1027         ID id, nid;
1028
1029         /* Freshly allocated, ignore it */
1030         if ( !ic->head && ic->count <= BDB_IDL_DB_SIZE ) {
1031                 return 0;
1032         }
1033
1034         rc = db->cursor( db, NULL, &curs, 0 );
1035         if ( rc )
1036                 return -1;
1037
1038         DBTzero( &key );
1039         DBTzero( &data );
1040
1041         bv2DBT( &ic->kstr, &key );
1042
1043         data.size = data.ulen = sizeof( ID );
1044         data.flags = DB_DBT_USERMEM;
1045         data.data = &nid;
1046
1047         rc = curs->c_get( curs, &key, &data, DB_SET );
1048         /* If key already exists and we're writing a range... */
1049         if ( rc == 0 && ic->count > BDB_IDL_DB_SIZE ) {
1050                 /* If it's not currently a range, must delete old info */
1051                 if ( nid ) {
1052                         /* Skip lo */
1053                         while ( curs->c_get( curs, &key, &data, DB_NEXT_DUP ) == 0 )
1054                                 curs->c_del( curs, 0 );
1055
1056                         nid = 0;
1057                         /* Store range marker */
1058                         curs->c_put( curs, &key, &data, DB_KEYFIRST );
1059                 } else {
1060
1061                         /* Skip lo */
1062                         rc = curs->c_get( curs, &key, &data, DB_NEXT_DUP );
1063
1064                         /* Get hi */
1065                         rc = curs->c_get( curs, &key, &data, DB_NEXT_DUP );
1066
1067                         /* Delete hi */
1068                         curs->c_del( curs, 0 );
1069                 }
1070                 BDB_ID2DISK( ic->last, &nid );
1071                 curs->c_put( curs, &key, &data, DB_KEYLAST );
1072                 rc = 0;
1073         } else if ( rc && rc != DB_NOTFOUND ) {
1074                 rc = -1;
1075         } else if ( ic->count > BDB_IDL_DB_SIZE ) {
1076                 /* range, didn't exist before */
1077                 nid = 0;
1078                 rc = curs->c_put( curs, &key, &data, DB_KEYLAST );
1079                 if ( rc == 0 ) {
1080                         BDB_ID2DISK( ic->first, &nid );
1081                         rc = curs->c_put( curs, &key, &data, DB_KEYLAST );
1082                         if ( rc == 0 ) {
1083                                 BDB_ID2DISK( ic->last, &nid );
1084                                 rc = curs->c_put( curs, &key, &data, DB_KEYLAST );
1085                         }
1086                 }
1087                 if ( rc ) {
1088                         rc = -1;
1089                 }
1090         } else {
1091                 int n;
1092
1093                 /* Just a normal write */
1094                 rc = 0;
1095                 for ( ice = ic->head, n=0; ice; ice = ice->next, n++ ) {
1096                         int end;
1097                         if ( ice->next ) {
1098                                 end = IDBLOCK;
1099                         } else {
1100                                 end = ic->count & (IDBLOCK-1);
1101                                 if ( !end )
1102                                         end = IDBLOCK;
1103                         }
1104                         for ( i=0; i<end; i++ ) {
1105                                 if ( !ice->ids[i] ) continue;
1106                                 BDB_ID2DISK( ice->ids[i], &nid );
1107                                 rc = curs->c_put( curs, &key, &data, DB_NODUPDATA );
1108                                 if ( rc ) {
1109                                         if ( rc == DB_KEYEXIST ) {
1110                                                 rc = 0;
1111                                                 continue;
1112                                         }
1113                                         rc = -1;
1114                                         break;
1115                                 }
1116                         }
1117                         if ( rc ) {
1118                                 rc = -1;
1119                                 break;
1120                         }
1121                 }
1122                 if ( ic->head ) {
1123                         ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
1124                         ic->tail->next = bdb_tool_idl_free_list;
1125                         bdb_tool_idl_free_list = ic->head;
1126                         bdb->bi_idl_cache_size -= n;
1127                         ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
1128                 }
1129         }
1130         if ( ic != db->app_private ) {
1131                 ch_free( ic );
1132         } else {
1133                 ic->head = ic->tail = NULL;
1134         }
1135         curs->c_close( curs );
1136         return rc;
1137 }
1138
1139 static int
1140 bdb_tool_idl_flush_db( DB *db, bdb_tool_idl_cache *ic )
1141 {
1142         Avlnode *root = db->app_private;
1143         int rc;
1144
1145         db->app_private = ic;
1146         rc = avl_apply( root, bdb_tool_idl_flush_one, db, -1, AVL_INORDER );
1147         avl_free( root, NULL );
1148         db->app_private = NULL;
1149         if ( rc != -1 )
1150                 rc = 0;
1151         return rc;
1152 }
1153
1154 static int
1155 bdb_tool_idl_flush( BackendDB *be )
1156 {
1157         struct bdb_info *bdb = (struct bdb_info *) be->be_private;
1158         DB *db;
1159         Avlnode *root;
1160         int i, rc = 0;
1161
1162         for ( i=BDB_NDB; i < bdb->bi_ndatabases; i++ ) {
1163                 db = bdb->bi_databases[i]->bdi_db;
1164                 if ( !db->app_private ) continue;
1165                 rc = bdb_tool_idl_flush_db( db, NULL );
1166                 if ( rc )
1167                         break;
1168         }
1169         if ( !rc ) {
1170                 bdb->bi_idl_cache_size = 0;
1171         }
1172         return rc;
1173 }
1174
1175 int bdb_tool_idl_add(
1176         BackendDB *be,
1177         DB *db,
1178         DB_TXN *txn,
1179         DBT *key,
1180         ID id )
1181 {
1182         struct bdb_info *bdb = (struct bdb_info *) be->be_private;
1183         bdb_tool_idl_cache *ic, itmp;
1184         bdb_tool_idl_cache_entry *ice;
1185         int rc;
1186
1187         if ( !bdb->bi_idl_cache_max_size )
1188                 return bdb_idl_insert_key( be, db, txn, key, id );
1189
1190         DBT2bv( key, &itmp.kstr );
1191
1192         ic = avl_find( (Avlnode *)db->app_private, &itmp, bdb_tool_idl_cmp );
1193
1194         /* No entry yet, create one */
1195         if ( !ic ) {
1196                 DBC *curs;
1197                 DBT data;
1198                 ID nid;
1199                 int rc;
1200
1201                 ic = ch_malloc( sizeof( bdb_tool_idl_cache ) + itmp.kstr.bv_len );
1202                 ic->kstr.bv_len = itmp.kstr.bv_len;
1203                 ic->kstr.bv_val = (char *)(ic+1);
1204                 AC_MEMCPY( ic->kstr.bv_val, itmp.kstr.bv_val, ic->kstr.bv_len );
1205                 ic->head = ic->tail = NULL;
1206                 ic->last = 0;
1207                 ic->count = 0;
1208                 avl_insert( (Avlnode **)&db->app_private, ic, bdb_tool_idl_cmp,
1209                         avl_dup_error );
1210
1211                 /* load existing key count here */
1212                 rc = db->cursor( db, NULL, &curs, 0 );
1213                 if ( rc ) return rc;
1214
1215                 data.ulen = sizeof( ID );
1216                 data.flags = DB_DBT_USERMEM;
1217                 data.data = &nid;
1218                 rc = curs->c_get( curs, key, &data, DB_SET );
1219                 if ( rc == 0 ) {
1220                         if ( nid == 0 ) {
1221                                 ic->count = BDB_IDL_DB_SIZE+1;
1222                         } else {
1223                                 db_recno_t count;
1224
1225                                 curs->c_count( curs, &count, 0 );
1226                                 ic->count = count;
1227                                 BDB_DISK2ID( &nid, &ic->first );
1228                         }
1229                 }
1230                 curs->c_close( curs );
1231         }
1232         /* are we a range already? */
1233         if ( ic->count > BDB_IDL_DB_SIZE ) {
1234                 ic->last = id;
1235                 return 0;
1236         /* Are we at the limit, and converting to a range? */
1237         } else if ( ic->count == BDB_IDL_DB_SIZE ) {
1238                 int n;
1239                 for ( ice = ic->head, n=0; ice; ice = ice->next, n++ )
1240                         /* counting */ ;
1241                 if ( n ) {
1242                         ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
1243                         ic->tail->next = bdb_tool_idl_free_list;
1244                         bdb_tool_idl_free_list = ic->head;
1245                         bdb->bi_idl_cache_size -= n;
1246                         ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
1247                 }
1248                 ic->head = ic->tail = NULL;
1249                 ic->last = id;
1250                 ic->count++;
1251                 return 0;
1252         }
1253         /* No free block, create that too */
1254         if ( !ic->tail || ( ic->count & (IDBLOCK-1)) == 0) {
1255                 ice = NULL;
1256                 ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
1257                 if ( bdb->bi_idl_cache_size >= bdb->bi_idl_cache_max_size ) {
1258                         ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
1259                         rc = bdb_tool_idl_flush_db( db, ic );
1260                         if ( rc )
1261                                 return rc;
1262                         avl_insert( (Avlnode **)&db->app_private, ic, bdb_tool_idl_cmp,
1263                                 avl_dup_error );
1264                         ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
1265                 }
1266                 bdb->bi_idl_cache_size++;
1267                 if ( bdb_tool_idl_free_list ) {
1268                         ice = bdb_tool_idl_free_list;
1269                         bdb_tool_idl_free_list = ice->next;
1270                 }
1271                 ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
1272                 if ( !ice ) {
1273                         ice = ch_malloc( sizeof( bdb_tool_idl_cache_entry ));
1274                 }
1275                 memset( ice, 0, sizeof( *ice ));
1276                 if ( !ic->head ) {
1277                         ic->head = ice;
1278                 } else {
1279                         ic->tail->next = ice;
1280                 }
1281                 ic->tail = ice;
1282                 if ( !ic->count )
1283                         ic->first = id;
1284         }
1285         ice = ic->tail;
1286         ice->ids[ ic->count & (IDBLOCK-1) ] = id;
1287         ic->count++;
1288
1289         return 0;
1290 }
1291 #endif
1292
1293 #ifdef USE_TRICKLE
1294 static void *
1295 bdb_tool_trickle_task( void *ctx, void *ptr )
1296 {
1297         DB_ENV *env = ptr;
1298         int wrote;
1299
1300         ldap_pvt_thread_mutex_lock( &bdb_tool_trickle_mutex );
1301         bdb_tool_trickle_active = 1;
1302         ldap_pvt_thread_cond_signal( &bdb_tool_trickle_cond_end );
1303         while ( 1 ) {
1304                 ldap_pvt_thread_cond_wait( &bdb_tool_trickle_cond,
1305                         &bdb_tool_trickle_mutex );
1306                 if ( slapd_shutdown )
1307                         break;
1308                 env->memp_trickle( env, 30, &wrote );
1309         }
1310         bdb_tool_trickle_active = 0;
1311         ldap_pvt_thread_cond_signal( &bdb_tool_trickle_cond_end );
1312         ldap_pvt_thread_mutex_unlock( &bdb_tool_trickle_mutex );
1313
1314         return NULL;
1315 }
1316 #endif
1317
1318 static void *
1319 bdb_tool_index_task( void *ctx, void *ptr )
1320 {
1321         int base = *(int *)ptr;
1322
1323         free( ptr );
1324         while ( 1 ) {
1325                 ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex );
1326                 bdb_tool_index_tcount--;
1327                 if ( !bdb_tool_index_tcount )
1328                         ldap_pvt_thread_cond_signal( &bdb_tool_index_cond_main );
1329                 ldap_pvt_thread_cond_wait( &bdb_tool_index_cond_work,
1330                         &bdb_tool_index_mutex );
1331                 if ( slapd_shutdown ) {
1332                         bdb_tool_index_tcount--;
1333                         if ( !bdb_tool_index_tcount )
1334                                 ldap_pvt_thread_cond_signal( &bdb_tool_index_cond_main );
1335                         ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex );
1336                         break;
1337                 }
1338                 ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex );
1339
1340                 bdb_tool_index_threads[base] = bdb_index_recrun( bdb_tool_ix_op,
1341                         bdb_tool_info, bdb_tool_index_rec, bdb_tool_ix_id, base );
1342         }
1343
1344         return NULL;
1345 }