git.sur5r.net Git - openldap/blob - servers/slapd/back-bdb/tools.c

   1 /* tools.c - tools for slap tools */
   2 /* $OpenLDAP$ */
   3 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
   4  *
   5  * Copyright 2000-2005 The OpenLDAP Foundation.
   6  * All rights reserved.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted only as authorized by the OpenLDAP
  10  * Public License.
  11  *
  12  * A copy of this license is available in the file LICENSE in the
  13  * top-level directory of the distribution or, alternatively, at
  14  * <http://www.OpenLDAP.org/license.html>.
  15  */
  16
  17 #include "portable.h"
  18
  19 #include <stdio.h>
  20 #include <ac/string.h>
  21
  22 #define AVL_INTERNAL
  23 #include "back-bdb.h"
  24 #include "idl.h"
  25
  26 static DBC *cursor = NULL;
  27 static DBT key, data;
  28
  29 typedef struct dn_id {
  30         ID id;
  31         struct berval dn;
  32 } dn_id;
  33
  34 #define HOLE_SIZE       4096
  35 static dn_id hbuf[HOLE_SIZE], *holes = hbuf;
  36 static unsigned nhmax = HOLE_SIZE;
  37 static unsigned nholes;
  38
  39 static int index_nattrs;
  40
  41 #ifdef BDB_TOOL_IDL_CACHING
  42 #define bdb_tool_idl_cmp                BDB_SYMBOL(tool_idl_cmp)
  43 #define bdb_tool_idl_flush_one          BDB_SYMBOL(tool_idl_flush_one)
  44 #define bdb_tool_idl_flush              BDB_SYMBOL(tool_idl_flush)
  45
  46 static int bdb_tool_idl_flush( BackendDB *be );
  47
  48 #define IDBLOCK 1024
  49
  50 typedef struct bdb_tool_idl_cache_entry {
  51         struct bdb_tool_idl_cache_entry *next;
  52         ID ids[IDBLOCK];
  53 } bdb_tool_idl_cache_entry;
  54
  55 typedef struct bdb_tool_idl_cache {
  56         struct berval kstr;
  57         bdb_tool_idl_cache_entry *head, *tail;
  58         ID first, last;
  59         int count;
  60 } bdb_tool_idl_cache;
  61
  62 static bdb_tool_idl_cache_entry *bdb_tool_idl_free_list;
  63 #endif  /* BDB_TOOL_IDL_CACHING */
  64
  65 static ID bdb_tool_ix_id;
  66 static Operation *bdb_tool_ix_op;
  67 static int *bdb_tool_index_threads, bdb_tool_index_tcount;
  68 static void *bdb_tool_index_rec;
  69 static struct bdb_info *bdb_tool_info;
  70 static ldap_pvt_thread_mutex_t bdb_tool_index_mutex;
  71 static ldap_pvt_thread_cond_t bdb_tool_index_cond;
  72
  73 static void * bdb_tool_index_task( void *ctx, void *ptr );
  74
  75 int bdb_tool_entry_open(
  76         BackendDB *be, int mode )
  77 {
  78         struct bdb_info *bdb = (struct bdb_info *) be->be_private;
  79
  80         /* initialize key and data thangs */
  81         DBTzero( &key );
  82         DBTzero( &data );
  83         key.flags = DB_DBT_REALLOC;
  84         data.flags = DB_DBT_REALLOC;
  85
  86         if (cursor == NULL) {
  87                 int rc = bdb->bi_id2entry->bdi_db->cursor(
  88                         bdb->bi_id2entry->bdi_db, NULL, &cursor,
  89                         bdb->bi_db_opflags );
  90                 if( rc != 0 ) {
  91                         return -1;
  92                 }
  93         }
  94
  95         /* Set up for threaded slapindex */
  96         if (( slapMode & (SLAP_TOOL_QUICK|SLAP_TOOL_READONLY)) == SLAP_TOOL_QUICK) {
  97                 if ( !bdb_tool_info ) {
  98                         int i;
  99                         ldap_pvt_thread_mutex_init( &bdb_tool_index_mutex );
 100                         ldap_pvt_thread_cond_init( &bdb_tool_index_cond );
 101                         bdb_tool_index_threads = ch_malloc( slap_tool_thread_max * sizeof( int ));
 102                         bdb_tool_index_rec = ch_malloc( bdb->bi_nattrs * sizeof( IndexRec ));
 103                         bdb_tool_index_tcount = slap_tool_thread_max - 1;
 104                         for (i=1; i<slap_tool_thread_max; i++) {
 105                                 int *ptr = ch_malloc( sizeof( int ));
 106                                 *ptr = i;
 107                                 ldap_pvt_thread_pool_submit( &connection_pool,
 108                                         bdb_tool_index_task, ptr );
 109                         }
 110                 }
 111                 bdb_tool_info = bdb;
 112         }
 113
 114         return 0;
 115 }
 116
 117 int bdb_tool_entry_close(
 118         BackendDB *be )
 119 {
 120         if ( bdb_tool_info ) {
 121                 slapd_shutdown = 1;
 122                 ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex );
 123                 bdb_tool_index_tcount = slap_tool_thread_max - 1;
 124                 ldap_pvt_thread_cond_broadcast( &bdb_tool_index_cond );
 125                 ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex );
 126         }
 127
 128         if( key.data ) {
 129                 ch_free( key.data );
 130                 key.data = NULL;
 131         }
 132         if( data.data ) {
 133                 ch_free( data.data );
 134                 data.data = NULL;
 135         }
 136
 137         if( cursor ) {
 138                 cursor->c_close( cursor );
 139                 cursor = NULL;
 140         }
 141
 142 #ifdef BDB_TOOL_IDL_CACHING
 143         bdb_tool_idl_flush( be );
 144 #endif
 145
 146         if( nholes ) {
 147                 unsigned i;
 148                 fprintf( stderr, "Error, entries missing!\n");
 149                 for (i=0; i<nholes; i++) {
 150                         fprintf(stderr, "  entry %ld: %s\n",
 151                                 holes[i].id, holes[i].dn.bv_val);
 152                 }
 153                 return -1;
 154         }
 155
 156         return 0;
 157 }
 158
 159 ID bdb_tool_entry_next(
 160         BackendDB *be )
 161 {
 162         int rc;
 163         ID id;
 164         struct bdb_info *bdb = (struct bdb_info *) be->be_private;
 165
 166         assert( be != NULL );
 167         assert( slapMode & SLAP_TOOL_MODE );
 168         assert( bdb != NULL );
 169
 170         rc = cursor->c_get( cursor, &key, &data, DB_NEXT );
 171
 172         if( rc != 0 ) {
 173                 /* If we're doing linear indexing and there are more attrs to
 174                  * index, and we're at the end of the database, start over.
 175                  */
 176                 if ( index_nattrs && rc == DB_NOTFOUND ) {
 177                         /* optional - do a checkpoint here? */
 178                         bdb_attr_info_free( bdb->bi_attrs[0] );
 179                         bdb->bi_attrs[0] = bdb->bi_attrs[index_nattrs];
 180                         index_nattrs--;
 181                         rc = cursor->c_get( cursor, &key, &data, DB_FIRST );
 182                         if ( rc ) {
 183                                 return NOID;
 184                         }
 185                 } else {
 186                         return NOID;
 187                 }
 188         }
 189
 190         if( data.data == NULL ) {
 191                 return NOID;
 192         }
 193
 194         BDB_DISK2ID( key.data, &id );
 195         return id;
 196 }
 197
 198 ID bdb_tool_dn2id_get(
 199         Backend *be,
 200         struct berval *dn
 201 )
 202 {
 203         Operation op = {0};
 204         Opheader ohdr = {0};
 205         EntryInfo *ei = NULL;
 206         int rc;
 207
 208         if ( BER_BVISEMPTY(dn) )
 209                 return 0;
 210
 211         op.o_hdr = &ohdr;
 212         op.o_bd = be;
 213         op.o_tmpmemctx = NULL;
 214         op.o_tmpmfuncs = &ch_mfuncs;
 215
 216         rc = bdb_cache_find_ndn( &op, NULL, dn, &ei );
 217         if ( ei ) bdb_cache_entryinfo_unlock( ei );
 218         if ( rc == DB_NOTFOUND )
 219                 return NOID;
 220
 221         return ei->bei_id;
 222 }
 223
 224 int bdb_tool_id2entry_get(
 225         Backend *be,
 226         ID id,
 227         Entry **e
 228 )
 229 {
 230         int rc = bdb_id2entry( be, NULL, 0, id, e );
 231
 232         if ( rc == DB_NOTFOUND && id == 0 ) {
 233                 Entry *dummy = ch_calloc( 1, sizeof(Entry) );
 234                 struct berval gluebv = BER_BVC("glue");
 235                 dummy->e_name.bv_val = ch_strdup( "" );
 236                 dummy->e_nname.bv_val = ch_strdup( "" );
 237                 attr_merge_one( dummy, slap_schema.si_ad_objectClass, &gluebv, NULL );
 238                 attr_merge_one( dummy, slap_schema.si_ad_structuralObjectClass,
 239                         &gluebv, NULL );
 240                 *e = dummy;
 241                 rc = LDAP_SUCCESS;
 242         }
 243         return rc;
 244 }
 245
 246 Entry* bdb_tool_entry_get( BackendDB *be, ID id )
 247 {
 248         int rc;
 249         Entry *e = NULL;
 250         struct berval bv;
 251
 252         assert( be != NULL );
 253         assert( slapMode & SLAP_TOOL_MODE );
 254         assert( data.data != NULL );
 255
 256         DBT2bv( &data, &bv );
 257
 258 #ifdef SLAP_ZONE_ALLOC
 259         /* FIXME: will add ctx later */
 260         rc = entry_decode( &bv, &e, NULL );
 261 #else
 262         rc = entry_decode( &bv, &e );
 263 #endif
 264
 265         if( rc == LDAP_SUCCESS ) {
 266                 e->e_id = id;
 267         }
 268 #ifdef BDB_HIER
 269         if ( slapMode & SLAP_TOOL_READONLY ) {
 270                 EntryInfo *ei = NULL;
 271                 Operation op = {0};
 272                 Opheader ohdr = {0};
 273
 274                 op.o_hdr = &ohdr;
 275                 op.o_bd = be;
 276                 op.o_tmpmemctx = NULL;
 277                 op.o_tmpmfuncs = &ch_mfuncs;
 278
 279                 rc = bdb_cache_find_parent( &op, NULL, cursor->locker, id, &ei );
 280                 if ( rc == LDAP_SUCCESS ) {
 281                         bdb_cache_entryinfo_unlock( ei );
 282                         e->e_private = ei;
 283                         ei->bei_e = e;
 284                         bdb_fix_dn( e, 0 );
 285                         ei->bei_e = NULL;
 286                         e->e_private = NULL;
 287                 }
 288         }
 289 #endif
 290         return e;
 291 }
 292
 293 static int bdb_tool_next_id(
 294         Operation *op,
 295         DB_TXN *tid,
 296         Entry *e,
 297         struct berval *text,
 298         int hole )
 299 {
 300         struct berval dn = e->e_name;
 301         struct berval ndn = e->e_nname;
 302         struct berval pdn, npdn;
 303         EntryInfo *ei = NULL, eidummy;
 304         int rc;
 305
 306         if (ndn.bv_len == 0) {
 307                 e->e_id = 0;
 308                 return 0;
 309         }
 310
 311         rc = bdb_cache_find_ndn( op, tid, &ndn, &ei );
 312         if ( ei ) bdb_cache_entryinfo_unlock( ei );
 313         if ( rc == DB_NOTFOUND ) {
 314                 if ( !be_issuffix( op->o_bd, &ndn ) ) {
 315                         ID eid = e->e_id;
 316                         dnParent( &dn, &pdn );
 317                         dnParent( &ndn, &npdn );
 318                         e->e_name = pdn;
 319                         e->e_nname = npdn;
 320                         rc = bdb_tool_next_id( op, tid, e, text, 1 );
 321                         e->e_name = dn;
 322                         e->e_nname = ndn;
 323                         if ( rc ) {
 324                                 return rc;
 325                         }
 326                         /* If parent didn't exist, it was created just now
 327                          * and its ID is now in e->e_id. Make sure the current
 328                          * entry gets added under the new parent ID.
 329                          */
 330                         if ( eid != e->e_id ) {
 331                                 eidummy.bei_id = e->e_id;
 332                                 ei = &eidummy;
 333                         }
 334                 }
 335                 rc = bdb_next_id( op->o_bd, tid, &e->e_id );
 336                 if ( rc ) {
 337                         snprintf( text->bv_val, text->bv_len,
 338                                 "next_id failed: %s (%d)",
 339                                 db_strerror(rc), rc );
 340                 Debug( LDAP_DEBUG_ANY,
 341                         "=> bdb_tool_next_id: %s\n", text->bv_val, 0, 0 );
 342                         return rc;
 343                 }
 344                 rc = bdb_dn2id_add( op, tid, ei, e );
 345                 if ( rc ) {
 346                         snprintf( text->bv_val, text->bv_len,
 347                                 "dn2id_add failed: %s (%d)",
 348                                 db_strerror(rc), rc );
 349                 Debug( LDAP_DEBUG_ANY,
 350                         "=> bdb_tool_next_id: %s\n", text->bv_val, 0, 0 );
 351                 } else if ( hole ) {
 352                         if ( nholes == nhmax - 1 ) {
 353                                 if ( holes == hbuf ) {
 354                                         holes = ch_malloc( nhmax * sizeof(dn_id) * 2 );
 355                                         AC_MEMCPY( holes, hbuf, sizeof(hbuf) );
 356                                 } else {
 357                                         holes = ch_realloc( holes, nhmax * sizeof(dn_id) * 2 );
 358                                 }
 359                                 nhmax *= 2;
 360                         }
 361                         ber_dupbv( &holes[nholes].dn, &ndn );
 362                         holes[nholes++].id = e->e_id;
 363                 }
 364         } else if ( !hole ) {
 365                 unsigned i;
 366
 367                 e->e_id = ei->bei_id;
 368
 369                 for ( i=0; i<nholes; i++) {
 370                         if ( holes[i].id == e->e_id ) {
 371                                 int j;
 372                                 free(holes[i].dn.bv_val);
 373                                 for (j=i;j<nholes;j++) holes[j] = holes[j+1];
 374                                 holes[j].id = 0;
 375                                 nholes--;
 376                                 break;
 377                         } else if ( holes[i].id > e->e_id ) {
 378                                 break;
 379                         }
 380                 }
 381         }
 382         return rc;
 383 }
 384
 385 static int
 386 bdb_tool_index_add(
 387         Operation *op,
 388         DB_TXN *txn,
 389         Entry *e )
 390 {
 391         struct bdb_info *bdb = (struct bdb_info *) op->o_bd->be_private;
 392
 393         if ( slapMode & SLAP_TOOL_QUICK ) {
 394                 IndexRec *ir;
 395                 int i, rc;
 396                 Attribute *a;
 397
 398                 ir = bdb_tool_index_rec;
 399                 memset(ir, 0, bdb->bi_nattrs * sizeof( IndexRec ));
 400
 401                 for ( a = e->e_attrs; a != NULL; a = a->a_next ) {
 402                         rc = bdb_index_recset( bdb, a, a->a_desc->ad_type,
 403                                 &a->a_desc->ad_tags, ir );
 404                         if ( rc )
 405                                 return rc;
 406                 }
 407                 bdb_tool_ix_id = e->e_id;
 408                 bdb_tool_ix_op = op;
 409                 ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex );
 410                 /* Wait for all threads to be ready */
 411                 while ( bdb_tool_index_tcount ) {
 412                         ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex );
 413                         ldap_pvt_thread_yield();
 414                         ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex );
 415                 }
 416                 for ( i=1; i<slap_tool_thread_max; i++ )
 417                         bdb_tool_index_threads[i] = LDAP_BUSY;
 418                 bdb_tool_index_tcount = slap_tool_thread_max - 1;
 419                 ldap_pvt_thread_cond_broadcast( &bdb_tool_index_cond );
 420                 ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex );
 421                 rc = bdb_index_recrun( op, bdb, ir, e->e_id, 0 );
 422                 if ( rc )
 423                         return rc;
 424                 for ( i=1; i<slap_tool_thread_max; i++ ) {
 425                         if ( bdb_tool_index_threads[i] == LDAP_BUSY ) {
 426                                 ldap_pvt_thread_yield();
 427                                 i--;
 428                                 continue;
 429                         }
 430                         if ( bdb_tool_index_threads[i] )
 431                                 return bdb_tool_index_threads[i];
 432                 }
 433                 return 0;
 434         } else {
 435                 return bdb_index_entry_add( op, txn, e );
 436         }
 437 }
 438
 439 ID bdb_tool_entry_put(
 440         BackendDB *be,
 441         Entry *e,
 442         struct berval *text )
 443 {
 444         int rc;
 445         struct bdb_info *bdb = (struct bdb_info *) be->be_private;
 446         DB_TXN *tid = NULL;
 447         Operation op = {0};
 448         Opheader ohdr = {0};
 449
 450         assert( be != NULL );
 451         assert( slapMode & SLAP_TOOL_MODE );
 452
 453         assert( text != NULL );
 454         assert( text->bv_val != NULL );
 455         assert( text->bv_val[0] == '\0' );      /* overconservative? */
 456
 457         Debug( LDAP_DEBUG_TRACE, "=> " LDAP_XSTRING(bdb_tool_entry_put)
 458                 "( %ld, \"%s\" )\n", (long) e->e_id, e->e_dn, 0 );
 459
 460         if (! (slapMode & SLAP_TOOL_QUICK)) {
 461         rc = TXN_BEGIN( bdb->bi_dbenv, NULL, &tid,
 462                 bdb->bi_db_opflags );
 463         if( rc != 0 ) {
 464                 snprintf( text->bv_val, text->bv_len,
 465                         "txn_begin failed: %s (%d)",
 466                         db_strerror(rc), rc );
 467                 Debug( LDAP_DEBUG_ANY,
 468                         "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
 469                          text->bv_val, 0, 0 );
 470                 return NOID;
 471         }
 472         }
 473
 474         op.o_hdr = &ohdr;
 475         op.o_bd = be;
 476         op.o_tmpmemctx = NULL;
 477         op.o_tmpmfuncs = &ch_mfuncs;
 478
 479         /* add dn2id indices */
 480         rc = bdb_tool_next_id( &op, tid, e, text, 0 );
 481         if( rc != 0 ) {
 482                 goto done;
 483         }
 484
 485         /* id2entry index */
 486         rc = bdb_id2entry_add( be, tid, e );
 487         if( rc != 0 ) {
 488                 snprintf( text->bv_val, text->bv_len,
 489                                 "id2entry_add failed: %s (%d)",
 490                                 db_strerror(rc), rc );
 491                 Debug( LDAP_DEBUG_ANY,
 492                         "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
 493                         text->bv_val, 0, 0 );
 494                 goto done;
 495         }
 496
 497         if ( !bdb->bi_linear_index )
 498                 rc = bdb_tool_index_add( &op, tid, e );
 499         if( rc != 0 ) {
 500                 snprintf( text->bv_val, text->bv_len,
 501                                 "index_entry_add failed: %s (%d)",
 502                                 db_strerror(rc), rc );
 503                 Debug( LDAP_DEBUG_ANY,
 504                         "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
 505                         text->bv_val, 0, 0 );
 506                 goto done;
 507         }
 508
 509 done:
 510         if( rc == 0 ) {
 511                 if ( !( slapMode & SLAP_TOOL_QUICK )) {
 512                 rc = TXN_COMMIT( tid, 0 );
 513                 if( rc != 0 ) {
 514                         snprintf( text->bv_val, text->bv_len,
 515                                         "txn_commit failed: %s (%d)",
 516                                         db_strerror(rc), rc );
 517                         Debug( LDAP_DEBUG_ANY,
 518                                 "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
 519                                 text->bv_val, 0, 0 );
 520                         e->e_id = NOID;
 521                 }
 522                 }
 523
 524         } else {
 525                 if ( !( slapMode & SLAP_TOOL_QUICK )) {
 526                 TXN_ABORT( tid );
 527                 snprintf( text->bv_val, text->bv_len,
 528                         "txn_aborted! %s (%d)",
 529                         db_strerror(rc), rc );
 530                 Debug( LDAP_DEBUG_ANY,
 531                         "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
 532                         text->bv_val, 0, 0 );
 533                 }
 534                 e->e_id = NOID;
 535         }
 536
 537         return e->e_id;
 538 }
 539
 540 int bdb_tool_entry_reindex(
 541         BackendDB *be,
 542         ID id )
 543 {
 544         struct bdb_info *bi = (struct bdb_info *) be->be_private;
 545         int rc;
 546         Entry *e;
 547         DB_TXN *tid = NULL;
 548         Operation op = {0};
 549         Opheader ohdr = {0};
 550
 551         Debug( LDAP_DEBUG_ARGS,
 552                 "=> " LDAP_XSTRING(bdb_tool_entry_reindex) "( %ld )\n",
 553                 (long) id, 0, 0 );
 554
 555         /* No indexes configured, nothing to do. Could return an
 556          * error here to shortcut things.
 557          */
 558         if (!bi->bi_attrs) {
 559                 return 0;
 560         }
 561
 562         /* Get the first attribute to index */
 563         if (bi->bi_linear_index && !index_nattrs) {
 564                 index_nattrs = bi->bi_nattrs - 1;
 565                 bi->bi_nattrs = 1;
 566         }
 567
 568         e = bdb_tool_entry_get( be, id );
 569
 570         if( e == NULL ) {
 571                 Debug( LDAP_DEBUG_ANY,
 572                         LDAP_XSTRING(bdb_tool_entry_reindex)
 573                         ": could not locate id=%ld\n",
 574                         (long) id, 0, 0 );
 575                 return -1;
 576         }
 577
 578         if (! (slapMode & SLAP_TOOL_QUICK)) {
 579         rc = TXN_BEGIN( bi->bi_dbenv, NULL, &tid, bi->bi_db_opflags );
 580         if( rc != 0 ) {
 581                 Debug( LDAP_DEBUG_ANY,
 582                         "=> " LDAP_XSTRING(bdb_tool_entry_reindex) ": "
 583                         "txn_begin failed: %s (%d)\n",
 584                         db_strerror(rc), rc, 0 );
 585                 goto done;
 586         }
 587         }
 588
 589         /*
 590          * just (re)add them for now
 591          * assume that some other routine (not yet implemented)
 592          * will zap index databases
 593          *
 594          */
 595
 596         Debug( LDAP_DEBUG_TRACE,
 597                 "=> " LDAP_XSTRING(bdb_tool_entry_reindex) "( %ld, \"%s\" )\n",
 598                 (long) id, e->e_dn, 0 );
 599
 600         op.o_hdr = &ohdr;
 601         op.o_bd = be;
 602         op.o_tmpmemctx = NULL;
 603         op.o_tmpmfuncs = &ch_mfuncs;
 604
 605         rc = bdb_tool_index_add( &op, tid, e );
 606
 607 done:
 608         if( rc == 0 ) {
 609                 if (! (slapMode & SLAP_TOOL_QUICK)) {
 610                 rc = TXN_COMMIT( tid, 0 );
 611                 if( rc != 0 ) {
 612                         Debug( LDAP_DEBUG_ANY,
 613                                 "=> " LDAP_XSTRING(bdb_tool_entry_reindex)
 614                                 ": txn_commit failed: %s (%d)\n",
 615                                 db_strerror(rc), rc, 0 );
 616                         e->e_id = NOID;
 617                 }
 618                 }
 619
 620         } else {
 621                 if (! (slapMode & SLAP_TOOL_QUICK)) {
 622                 TXN_ABORT( tid );
 623                 Debug( LDAP_DEBUG_ANY,
 624                         "=> " LDAP_XSTRING(bdb_tool_entry_reindex)
 625                         ": txn_aborted! %s (%d)\n",
 626                         db_strerror(rc), rc, 0 );
 627                 }
 628                 e->e_id = NOID;
 629         }
 630         bdb_entry_release( &op, e, 0 );
 631
 632         return rc;
 633 }
 634
 635 ID bdb_tool_entry_modify(
 636         BackendDB *be,
 637         Entry *e,
 638         struct berval *text )
 639 {
 640         int rc;
 641         struct bdb_info *bdb = (struct bdb_info *) be->be_private;
 642         DB_TXN *tid = NULL;
 643         Operation op = {0};
 644         Opheader ohdr = {0};
 645
 646         assert( be != NULL );
 647         assert( slapMode & SLAP_TOOL_MODE );
 648
 649         assert( text != NULL );
 650         assert( text->bv_val != NULL );
 651         assert( text->bv_val[0] == '\0' );      /* overconservative? */
 652
 653         assert ( e->e_id != NOID );
 654
 655         Debug( LDAP_DEBUG_TRACE,
 656                 "=> " LDAP_XSTRING(bdb_tool_entry_modify) "( %ld, \"%s\" )\n",
 657                 (long) e->e_id, e->e_dn, 0 );
 658
 659         if (! (slapMode & SLAP_TOOL_QUICK)) {
 660         rc = TXN_BEGIN( bdb->bi_dbenv, NULL, &tid,
 661                 bdb->bi_db_opflags );
 662         if( rc != 0 ) {
 663                 snprintf( text->bv_val, text->bv_len,
 664                         "txn_begin failed: %s (%d)",
 665                         db_strerror(rc), rc );
 666                 Debug( LDAP_DEBUG_ANY,
 667                         "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n",
 668                          text->bv_val, 0, 0 );
 669                 return NOID;
 670         }
 671         }
 672
 673         op.o_hdr = &ohdr;
 674         op.o_bd = be;
 675         op.o_tmpmemctx = NULL;
 676         op.o_tmpmfuncs = &ch_mfuncs;
 677
 678         /* id2entry index */
 679         rc = bdb_id2entry_update( be, tid, e );
 680         if( rc != 0 ) {
 681                 snprintf( text->bv_val, text->bv_len,
 682                                 "id2entry_add failed: %s (%d)",
 683                                 db_strerror(rc), rc );
 684                 Debug( LDAP_DEBUG_ANY,
 685                         "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n",
 686                         text->bv_val, 0, 0 );
 687                 goto done;
 688         }
 689
 690 #if 0
 691         /* FIXME: this is bogus, we don't have the old values to delete
 692          * from the index because the given entry has already been modified.
 693          */
 694         rc = bdb_index_entry_del( &op, tid, e );
 695         if( rc != 0 ) {
 696                 snprintf( text->bv_val, text->bv_len,
 697                                 "index_entry_del failed: %s (%d)",
 698                                 db_strerror(rc), rc );
 699                 Debug( LDAP_DEBUG_ANY,
 700                         "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n",
 701                         text->bv_val, 0, 0 );
 702                 goto done;
 703         }
 704 #endif
 705
 706         rc = bdb_index_entry_add( &op, tid, e );
 707         if( rc != 0 ) {
 708                 snprintf( text->bv_val, text->bv_len,
 709                                 "index_entry_add failed: %s (%d)",
 710                                 db_strerror(rc), rc );
 711                 Debug( LDAP_DEBUG_ANY,
 712                         "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n",
 713                         text->bv_val, 0, 0 );
 714                 goto done;
 715         }
 716
 717 done:
 718         if( rc == 0 ) {
 719                 if (! (slapMode & SLAP_TOOL_QUICK)) {
 720                 rc = TXN_COMMIT( tid, 0 );
 721                 if( rc != 0 ) {
 722                         snprintf( text->bv_val, text->bv_len,
 723                                         "txn_commit failed: %s (%d)",
 724                                         db_strerror(rc), rc );
 725                         Debug( LDAP_DEBUG_ANY,
 726                                 "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": "
 727                                 "%s\n", text->bv_val, 0, 0 );
 728                         e->e_id = NOID;
 729                 }
 730                 }
 731
 732         } else {
 733                 if (! (slapMode & SLAP_TOOL_QUICK)) {
 734                 TXN_ABORT( tid );
 735                 snprintf( text->bv_val, text->bv_len,
 736                         "txn_aborted! %s (%d)",
 737                         db_strerror(rc), rc );
 738                 Debug( LDAP_DEBUG_ANY,
 739                         "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n",
 740                         text->bv_val, 0, 0 );
 741                 }
 742                 e->e_id = NOID;
 743         }
 744
 745         return e->e_id;
 746 }
 747
 748 #ifdef BDB_TOOL_IDL_CACHING
 749 static int
 750 bdb_tool_idl_cmp( const void *v1, const void *v2 )
 751 {
 752         const bdb_tool_idl_cache *c1 = v1, *c2 = v2;
 753         int rc;
 754
 755         if (( rc = c1->kstr.bv_len - c2->kstr.bv_len )) return rc;
 756         return memcmp( c1->kstr.bv_val, c2->kstr.bv_val, c1->kstr.bv_len );
 757 }
 758
 759 static int
 760 bdb_tool_idl_flush_one( void *v1, void *arg )
 761 {
 762         bdb_tool_idl_cache *ic = v1;
 763         DB *db = arg;
 764         struct bdb_info *bdb = bdb_tool_info;
 765         bdb_tool_idl_cache_entry *ice;
 766         DBC *curs;
 767         DBT key, data;
 768         int i, rc;
 769         ID id, nid;
 770
 771         /* Freshly allocated, ignore it */
 772         if ( !ic->head && ic->count <= BDB_IDL_DB_SIZE ) {
 773                 return 0;
 774         }
 775
 776         rc = db->cursor( db, NULL, &curs, 0 );
 777         if ( rc )
 778                 return -1;
 779
 780         DBTzero( &key );
 781         DBTzero( &data );
 782
 783         bv2DBT( &ic->kstr, &key );
 784
 785         data.size = data.ulen = sizeof( ID );
 786         data.flags = DB_DBT_USERMEM;
 787         data.data = &nid;
 788
 789         rc = curs->c_get( curs, &key, &data, DB_SET );
 790         /* If key already exists and we're writing a range... */
 791         if ( rc == 0 && ic->count > BDB_IDL_DB_SIZE ) {
 792                 /* If it's not currently a range, must delete old info */
 793                 if ( nid ) {
 794                         /* Skip lo */
 795                         while ( curs->c_get( curs, &key, &data, DB_NEXT_DUP ) == 0 )
 796                                 curs->c_del( curs, 0 );
 797
 798                         nid = 0;
 799                         /* Store range marker */
 800                         curs->c_put( curs, &key, &data, DB_KEYFIRST );
 801                 } else {
 802
 803                         /* Skip lo */
 804                         rc = curs->c_get( curs, &key, &data, DB_NEXT_DUP );
 805
 806                         /* Get hi */
 807                         rc = curs->c_get( curs, &key, &data, DB_NEXT_DUP );
 808
 809                         /* Delete hi */
 810                         curs->c_del( curs, 0 );
 811                 }
 812                 BDB_ID2DISK( ic->last, &nid );
 813                 curs->c_put( curs, &key, &data, DB_KEYLAST );
 814                 rc = 0;
 815         } else if ( rc && rc != DB_NOTFOUND ) {
 816                 rc = -1;
 817         } else if ( ic->count > BDB_IDL_DB_SIZE ) {
 818                 /* range, didn't exist before */
 819                 nid = 0;
 820                 rc = curs->c_put( curs, &key, &data, DB_KEYLAST );
 821                 if ( rc == 0 ) {
 822                         BDB_ID2DISK( ic->first, &nid );
 823                         rc = curs->c_put( curs, &key, &data, DB_KEYLAST );
 824                         if ( rc == 0 ) {
 825                                 BDB_ID2DISK( ic->last, &nid );
 826                                 rc = curs->c_put( curs, &key, &data, DB_KEYLAST );
 827                         }
 828                 }
 829                 if ( rc ) {
 830                         rc = -1;
 831                 }
 832         } else {
 833                 int n;
 834
 835                 /* Just a normal write */
 836                 rc = 0;
 837                 for ( ice = ic->head, n=0; ice; ice = ice->next, n++ ) {
 838                         int end;
 839                         if ( ice->next ) {
 840                                 end = IDBLOCK;
 841                         } else {
 842                                 end = ic->count & (IDBLOCK-1);
 843                                 if ( !end )
 844                                         end = IDBLOCK;
 845                         }
 846                         for ( i=0; i<end; i++ ) {
 847                                 if ( !ice->ids[i] ) continue;
 848                                 BDB_ID2DISK( ice->ids[i], &nid );
 849                                 rc = curs->c_put( curs, &key, &data, DB_NODUPDATA );
 850                                 if ( rc ) {
 851                                         if ( rc == DB_KEYEXIST ) {
 852                                                 rc = 0;
 853                                                 continue;
 854                                         }
 855                                         rc = -1;
 856                                         break;
 857                                 }
 858                         }
 859                         if ( rc ) {
 860                                 rc = -1;
 861                                 break;
 862                         }
 863                 }
 864                 if ( ic->head ) {
 865                         ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
 866                         ic->tail->next = bdb_tool_idl_free_list;
 867                         bdb_tool_idl_free_list = ic->head;
 868                         bdb->bi_idl_cache_size -= n;
 869                         ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
 870                 }
 871         }
 872         if ( ic != db->app_private ) {
 873                 ch_free( ic );
 874         } else {
 875                 ic->head = ic->tail = NULL;
 876         }
 877         curs->c_close( curs );
 878         return rc;
 879 }
 880
 881 static int
 882 bdb_tool_idl_flush_db( DB *db, bdb_tool_idl_cache *ic )
 883 {
 884         Avlnode *root = db->app_private;
 885         int rc;
 886
 887         db->app_private = ic;
 888         rc = avl_apply( root, bdb_tool_idl_flush_one, db, -1, AVL_INORDER );
 889         avl_free( root, NULL );
 890         db->app_private = NULL;
 891         if ( rc != -1 )
 892                 rc = 0;
 893         return rc;
 894 }
 895
 896 static int
 897 bdb_tool_idl_flush( BackendDB *be )
 898 {
 899         struct bdb_info *bdb = (struct bdb_info *) be->be_private;
 900         DB *db;
 901         Avlnode *root;
 902         int i, rc = 0;
 903
 904         for ( i=BDB_NDB; i < bdb->bi_ndatabases; i++ ) {
 905                 db = bdb->bi_databases[i]->bdi_db;
 906                 if ( !db->app_private ) continue;
 907                 rc = bdb_tool_idl_flush_db( db, NULL );
 908                 if ( rc )
 909                         break;
 910         }
 911         if ( !rc ) {
 912                 bdb->bi_idl_cache_size = 0;
 913         }
 914         return rc;
 915 }
 916
 917 int bdb_tool_idl_add(
 918         BackendDB *be,
 919         DB *db,
 920         DB_TXN *txn,
 921         DBT *key,
 922         ID id )
 923 {
 924         struct bdb_info *bdb = (struct bdb_info *) be->be_private;
 925         bdb_tool_idl_cache *ic, itmp;
 926         bdb_tool_idl_cache_entry *ice;
 927         int rc;
 928
 929         if ( !bdb->bi_idl_cache_max_size )
 930                 return bdb_idl_insert_key( be, db, txn, key, id );
 931
 932         DBT2bv( key, &itmp.kstr );
 933
 934         ic = avl_find( (Avlnode *)db->app_private, &itmp, bdb_tool_idl_cmp );
 935
 936         /* No entry yet, create one */
 937         if ( !ic ) {
 938                 DBC *curs;
 939                 DBT data;
 940                 ID nid;
 941                 int rc;
 942
 943                 ic = ch_malloc( sizeof( bdb_tool_idl_cache ) + itmp.kstr.bv_len );
 944                 ic->kstr.bv_len = itmp.kstr.bv_len;
 945                 ic->kstr.bv_val = (char *)(ic+1);
 946                 AC_MEMCPY( ic->kstr.bv_val, itmp.kstr.bv_val, ic->kstr.bv_len );
 947                 ic->head = ic->tail = NULL;
 948                 ic->last = 0;
 949                 ic->count = 0;
 950                 avl_insert( (Avlnode **)&db->app_private, ic, bdb_tool_idl_cmp,
 951                         avl_dup_error );
 952
 953                 /* load existing key count here */
 954                 rc = db->cursor( db, NULL, &curs, 0 );
 955                 if ( rc ) return rc;
 956
 957                 data.ulen = sizeof( ID );
 958                 data.flags = DB_DBT_USERMEM;
 959                 data.data = &nid;
 960                 rc = curs->c_get( curs, key, &data, DB_SET );
 961                 if ( rc == 0 ) {
 962                         if ( nid == 0 ) {
 963                                 ic->count = BDB_IDL_DB_SIZE+1;
 964                         } else {
 965                                 db_recno_t count;
 966
 967                                 curs->c_count( curs, &count, 0 );
 968                                 ic->count = count;
 969                                 BDB_DISK2ID( &nid, &ic->first );
 970                         }
 971                 }
 972                 curs->c_close( curs );
 973         }
 974         /* are we a range already? */
 975         if ( ic->count > BDB_IDL_DB_SIZE ) {
 976                 ic->last = id;
 977                 return 0;
 978         /* Are we at the limit, and converting to a range? */
 979         } else if ( ic->count == BDB_IDL_DB_SIZE ) {
 980                 int n;
 981                 for ( ice = ic->head, n=0; ice; ice = ice->next, n++ )
 982                         /* counting */ ;
 983                 if ( n ) {
 984                         ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
 985                         ic->tail->next = bdb_tool_idl_free_list;
 986                         bdb_tool_idl_free_list = ic->head;
 987                         bdb->bi_idl_cache_size -= n;
 988                         ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
 989                 }
 990                 ic->head = ic->tail = NULL;
 991                 ic->last = id;
 992                 ic->count++;
 993                 return 0;
 994         }
 995         /* No free block, create that too */
 996         if ( !ic->tail || ( ic->count & (IDBLOCK-1)) == 0) {
 997                 ice = NULL;
 998                 ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
 999                 if ( bdb->bi_idl_cache_size >= bdb->bi_idl_cache_max_size ) {
1000                         ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
1001                         rc = bdb_tool_idl_flush_db( db, ic );
1002                         if ( rc )
1003                                 return rc;
1004                         avl_insert( (Avlnode **)&db->app_private, ic, bdb_tool_idl_cmp,
1005                                 avl_dup_error );
1006                         ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
1007                 }
1008                 bdb->bi_idl_cache_size++;
1009                 if ( bdb_tool_idl_free_list ) {
1010                         ice = bdb_tool_idl_free_list;
1011                         bdb_tool_idl_free_list = ice->next;
1012                 }
1013                 ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
1014                 if ( !ice ) {
1015                         ice = ch_malloc( sizeof( bdb_tool_idl_cache_entry ));
1016                 }
1017                 memset( ice, 0, sizeof( *ice ));
1018                 if ( !ic->head ) {
1019                         ic->head = ice;
1020                 } else {
1021                         ic->tail->next = ice;
1022                 }
1023                 ic->tail = ice;
1024                 if ( !ic->count )
1025                         ic->first = id;
1026         }
1027         ice = ic->tail;
1028         ice->ids[ ic->count & (IDBLOCK-1) ] = id;
1029         ic->count++;
1030
1031         return 0;
1032 }
1033 #endif
1034
1035 static void *
1036 bdb_tool_index_task( void *ctx, void *ptr )
1037 {
1038         int base = *(int *)ptr;
1039
1040         free( ptr );
1041         while ( 1 ) {
1042                 ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex );
1043                 bdb_tool_index_tcount--;
1044                 ldap_pvt_thread_cond_wait( &bdb_tool_index_cond,
1045                         &bdb_tool_index_mutex );
1046                 ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex );
1047                 if ( slapd_shutdown )
1048                         break;
1049
1050                 bdb_tool_index_threads[base] = bdb_index_recrun( bdb_tool_ix_op,
1051                         bdb_tool_info, bdb_tool_index_rec, bdb_tool_ix_id, base );
1052         }
1053
1054         return NULL;
1055 }