git.sur5r.net Git - openldap/blob - servers/slapd/back-bdb/tools.c

   1 /* tools.c - tools for slap tools */
   2 /* $OpenLDAP$ */
   3 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
   4  *
   5  * Copyright 2000-2005 The OpenLDAP Foundation.
   6  * All rights reserved.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted only as authorized by the OpenLDAP
  10  * Public License.
  11  *
  12  * A copy of this license is available in the file LICENSE in the
  13  * top-level directory of the distribution or, alternatively, at
  14  * <http://www.OpenLDAP.org/license.html>.
  15  */
  16
  17 #include "portable.h"
  18
  19 #include <stdio.h>
  20 #include <ac/string.h>
  21
  22 #define AVL_INTERNAL
  23 #include "back-bdb.h"
  24 #include "idl.h"
  25
  26 static DBC *cursor = NULL;
  27 static DBT key, data;
  28
  29 typedef struct dn_id {
  30         ID id;
  31         struct berval dn;
  32 } dn_id;
  33
  34 #define HOLE_SIZE       4096
  35 static dn_id hbuf[HOLE_SIZE], *holes = hbuf;
  36 static unsigned nhmax = HOLE_SIZE;
  37 static unsigned nholes;
  38
  39 static int index_nattrs;
  40
  41 #ifdef BDB_TOOL_IDL_CACHING
  42 #define bdb_tool_idl_cmp                BDB_SYMBOL(tool_idl_cmp)
  43 #define bdb_tool_idl_flush_one          BDB_SYMBOL(tool_idl_flush_one)
  44 #define bdb_tool_idl_flush              BDB_SYMBOL(tool_idl_flush)
  45
  46 static int bdb_tool_idl_flush( BackendDB *be );
  47
  48 #define IDBLOCK 1024
  49
  50 typedef struct bdb_tool_idl_cache_entry {
  51         struct bdb_tool_idl_cache_entry *next;
  52         ID ids[IDBLOCK];
  53 } bdb_tool_idl_cache_entry;
  54
  55 typedef struct bdb_tool_idl_cache {
  56         struct berval kstr;
  57         bdb_tool_idl_cache_entry *head, *tail;
  58         ID first, last;
  59         int count;
  60 } bdb_tool_idl_cache;
  61
  62 static bdb_tool_idl_cache_entry *bdb_tool_idl_free_list;
  63 #endif  /* BDB_TOOL_IDL_CACHING */
  64
  65 static ID bdb_tool_ix_id;
  66 static Operation *bdb_tool_ix_op;
  67 static volatile int *bdb_tool_index_threads;
  68 static void *bdb_tool_index_rec;
  69 static struct bdb_info *bdb_tool_info;
  70 static ldap_pvt_thread_mutex_t bdb_tool_index_mutex;
  71 static ldap_pvt_thread_cond_t bdb_tool_index_cond;
  72
  73 static int bdb_tool_ix_rec( int base );
  74 static void * bdb_tool_index_task( void *ctx, void *ptr );
  75
  76 int bdb_tool_entry_open(
  77         BackendDB *be, int mode )
  78 {
  79         struct bdb_info *bdb = (struct bdb_info *) be->be_private;
  80
  81         /* initialize key and data thangs */
  82         DBTzero( &key );
  83         DBTzero( &data );
  84         key.flags = DB_DBT_REALLOC;
  85         data.flags = DB_DBT_REALLOC;
  86
  87         if (cursor == NULL) {
  88                 int rc = bdb->bi_id2entry->bdi_db->cursor(
  89                         bdb->bi_id2entry->bdi_db, NULL, &cursor,
  90                         bdb->bi_db_opflags );
  91                 if( rc != 0 ) {
  92                         return -1;
  93                 }
  94         }
  95
  96         /* Set up for threaded slapindex */
  97         if (( slapMode & (SLAP_TOOL_QUICK|SLAP_TOOL_READONLY)) == SLAP_TOOL_QUICK) {
  98                 if ( !bdb_tool_info ) {
  99                         int i;
 100                         ldap_pvt_thread_mutex_init( &bdb_tool_index_mutex );
 101                         ldap_pvt_thread_cond_init( &bdb_tool_index_cond );
 102                         bdb_tool_index_threads = ch_malloc( slap_tool_thread_max * sizeof( int ));
 103                         bdb_tool_index_rec = ch_malloc( bdb->bi_nattrs * sizeof( IndexRec ));
 104                         for (i=1; i<slap_tool_thread_max; i++) {
 105                                 int *ptr = ch_malloc( sizeof( int ));
 106                                 *ptr = i;
 107                                 ldap_pvt_thread_pool_submit( &connection_pool,
 108                                         bdb_tool_index_task, ptr );
 109                                 ldap_pvt_thread_yield();
 110                         }
 111                 }
 112                 bdb_tool_info = bdb;
 113         }
 114
 115         return 0;
 116 }
 117
 118 int bdb_tool_entry_close(
 119         BackendDB *be )
 120 {
 121         struct bdb_info *bdb = (struct bdb_info *) be->be_private;
 122
 123         if ( bdb_tool_info ) {
 124                 slapd_shutdown = 1;
 125                 ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex );
 126                 ldap_pvt_thread_cond_broadcast( &bdb_tool_index_cond );
 127                 ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex );
 128         }
 129
 130         if( key.data ) {
 131                 ch_free( key.data );
 132                 key.data = NULL;
 133         }
 134         if( data.data ) {
 135                 ch_free( data.data );
 136                 data.data = NULL;
 137         }
 138
 139         if( cursor ) {
 140                 cursor->c_close( cursor );
 141                 cursor = NULL;
 142         }
 143
 144 #ifdef BDB_TOOL_IDL_CACHING
 145         bdb_tool_idl_flush( be );
 146 #endif
 147
 148         if( nholes ) {
 149                 unsigned i;
 150                 fprintf( stderr, "Error, entries missing!\n");
 151                 for (i=0; i<nholes; i++) {
 152                         fprintf(stderr, "  entry %ld: %s\n",
 153                                 holes[i].id, holes[i].dn.bv_val);
 154                 }
 155                 return -1;
 156         }
 157
 158         return 0;
 159 }
 160
 161 ID bdb_tool_entry_next(
 162         BackendDB *be )
 163 {
 164         int rc;
 165         ID id;
 166         struct bdb_info *bdb = (struct bdb_info *) be->be_private;
 167
 168         assert( be != NULL );
 169         assert( slapMode & SLAP_TOOL_MODE );
 170         assert( bdb != NULL );
 171
 172         rc = cursor->c_get( cursor, &key, &data, DB_NEXT );
 173
 174         if( rc != 0 ) {
 175                 /* If we're doing linear indexing and there are more attrs to
 176                  * index, and we're at the end of the database, start over.
 177                  */
 178                 if ( index_nattrs && rc == DB_NOTFOUND ) {
 179                         /* optional - do a checkpoint here? */
 180                         bdb_attr_info_free( bdb->bi_attrs[0] );
 181                         bdb->bi_attrs[0] = bdb->bi_attrs[index_nattrs];
 182                         index_nattrs--;
 183                         rc = cursor->c_get( cursor, &key, &data, DB_FIRST );
 184                         if ( rc ) {
 185                                 return NOID;
 186                         }
 187                 } else {
 188                         return NOID;
 189                 }
 190         }
 191
 192         if( data.data == NULL ) {
 193                 return NOID;
 194         }
 195
 196         BDB_DISK2ID( key.data, &id );
 197         return id;
 198 }
 199
 200 ID bdb_tool_dn2id_get(
 201         Backend *be,
 202         struct berval *dn
 203 )
 204 {
 205         Operation op = {0};
 206         Opheader ohdr = {0};
 207         EntryInfo *ei = NULL;
 208         int rc;
 209
 210         if ( BER_BVISEMPTY(dn) )
 211                 return 0;
 212
 213         op.o_hdr = &ohdr;
 214         op.o_bd = be;
 215         op.o_tmpmemctx = NULL;
 216         op.o_tmpmfuncs = &ch_mfuncs;
 217
 218         rc = bdb_cache_find_ndn( &op, NULL, dn, &ei );
 219         if ( ei ) bdb_cache_entryinfo_unlock( ei );
 220         if ( rc == DB_NOTFOUND )
 221                 return NOID;
 222
 223         return ei->bei_id;
 224 }
 225
 226 int bdb_tool_id2entry_get(
 227         Backend *be,
 228         ID id,
 229         Entry **e
 230 )
 231 {
 232         int rc = bdb_id2entry( be, NULL, 0, id, e );
 233
 234         if ( rc == DB_NOTFOUND && id == 0 ) {
 235                 Entry *dummy = ch_calloc( 1, sizeof(Entry) );
 236                 struct berval gluebv = BER_BVC("glue");
 237                 dummy->e_name.bv_val = ch_strdup( "" );
 238                 dummy->e_nname.bv_val = ch_strdup( "" );
 239                 attr_merge_one( dummy, slap_schema.si_ad_objectClass, &gluebv, NULL );
 240                 attr_merge_one( dummy, slap_schema.si_ad_structuralObjectClass,
 241                         &gluebv, NULL );
 242                 *e = dummy;
 243                 rc = LDAP_SUCCESS;
 244         }
 245         return rc;
 246 }
 247
 248 Entry* bdb_tool_entry_get( BackendDB *be, ID id )
 249 {
 250         int rc;
 251         Entry *e = NULL;
 252         struct berval bv;
 253
 254         assert( be != NULL );
 255         assert( slapMode & SLAP_TOOL_MODE );
 256         assert( data.data != NULL );
 257
 258         DBT2bv( &data, &bv );
 259
 260 #ifdef SLAP_ZONE_ALLOC
 261         /* FIXME: will add ctx later */
 262         rc = entry_decode( &bv, &e, NULL );
 263 #else
 264         rc = entry_decode( &bv, &e );
 265 #endif
 266
 267         if( rc == LDAP_SUCCESS ) {
 268                 e->e_id = id;
 269         }
 270 #ifdef BDB_HIER
 271         if ( slapMode & SLAP_TOOL_READONLY ) {
 272                 EntryInfo *ei = NULL;
 273                 Operation op = {0};
 274                 Opheader ohdr = {0};
 275
 276                 op.o_hdr = &ohdr;
 277                 op.o_bd = be;
 278                 op.o_tmpmemctx = NULL;
 279                 op.o_tmpmfuncs = &ch_mfuncs;
 280
 281                 rc = bdb_cache_find_parent( &op, NULL, cursor->locker, id, &ei );
 282                 if ( rc == LDAP_SUCCESS ) {
 283                         bdb_cache_entryinfo_unlock( ei );
 284                         e->e_private = ei;
 285                         ei->bei_e = e;
 286                         bdb_fix_dn( e, 0 );
 287                         ei->bei_e = NULL;
 288                         e->e_private = NULL;
 289                 }
 290         }
 291 #endif
 292         return e;
 293 }
 294
 295 static int bdb_tool_next_id(
 296         Operation *op,
 297         DB_TXN *tid,
 298         Entry *e,
 299         struct berval *text,
 300         int hole )
 301 {
 302         struct berval dn = e->e_name;
 303         struct berval ndn = e->e_nname;
 304         struct berval pdn, npdn;
 305         EntryInfo *ei = NULL, eidummy;
 306         int rc;
 307
 308         if (ndn.bv_len == 0) {
 309                 e->e_id = 0;
 310                 return 0;
 311         }
 312
 313         rc = bdb_cache_find_ndn( op, tid, &ndn, &ei );
 314         if ( ei ) bdb_cache_entryinfo_unlock( ei );
 315         if ( rc == DB_NOTFOUND ) {
 316                 if ( !be_issuffix( op->o_bd, &ndn ) ) {
 317                         ID eid = e->e_id;
 318                         dnParent( &dn, &pdn );
 319                         dnParent( &ndn, &npdn );
 320                         e->e_name = pdn;
 321                         e->e_nname = npdn;
 322                         rc = bdb_tool_next_id( op, tid, e, text, 1 );
 323                         e->e_name = dn;
 324                         e->e_nname = ndn;
 325                         if ( rc ) {
 326                                 return rc;
 327                         }
 328                         /* If parent didn't exist, it was created just now
 329                          * and its ID is now in e->e_id. Make sure the current
 330                          * entry gets added under the new parent ID.
 331                          */
 332                         if ( eid != e->e_id ) {
 333                                 eidummy.bei_id = e->e_id;
 334                                 ei = &eidummy;
 335                         }
 336                 }
 337                 rc = bdb_next_id( op->o_bd, tid, &e->e_id );
 338                 if ( rc ) {
 339                         snprintf( text->bv_val, text->bv_len,
 340                                 "next_id failed: %s (%d)",
 341                                 db_strerror(rc), rc );
 342                 Debug( LDAP_DEBUG_ANY,
 343                         "=> bdb_tool_next_id: %s\n", text->bv_val, 0, 0 );
 344                         return rc;
 345                 }
 346                 rc = bdb_dn2id_add( op, tid, ei, e );
 347                 if ( rc ) {
 348                         snprintf( text->bv_val, text->bv_len,
 349                                 "dn2id_add failed: %s (%d)",
 350                                 db_strerror(rc), rc );
 351                 Debug( LDAP_DEBUG_ANY,
 352                         "=> bdb_tool_next_id: %s\n", text->bv_val, 0, 0 );
 353                 } else if ( hole ) {
 354                         if ( nholes == nhmax - 1 ) {
 355                                 if ( holes == hbuf ) {
 356                                         holes = ch_malloc( nhmax * sizeof(dn_id) * 2 );
 357                                         AC_MEMCPY( holes, hbuf, sizeof(hbuf) );
 358                                 } else {
 359                                         holes = ch_realloc( holes, nhmax * sizeof(dn_id) * 2 );
 360                                 }
 361                                 nhmax *= 2;
 362                         }
 363                         ber_dupbv( &holes[nholes].dn, &ndn );
 364                         holes[nholes++].id = e->e_id;
 365                 }
 366         } else if ( !hole ) {
 367                 unsigned i;
 368
 369                 e->e_id = ei->bei_id;
 370
 371                 for ( i=0; i<nholes; i++) {
 372                         if ( holes[i].id == e->e_id ) {
 373                                 int j;
 374                                 free(holes[i].dn.bv_val);
 375                                 for (j=i;j<nholes;j++) holes[j] = holes[j+1];
 376                                 holes[j].id = 0;
 377                                 nholes--;
 378                                 break;
 379                         } else if ( holes[i].id > e->e_id ) {
 380                                 break;
 381                         }
 382                 }
 383         }
 384         return rc;
 385 }
 386
 387 static int
 388 bdb_tool_index_add(
 389         Operation *op,
 390         DB_TXN *txn,
 391         Entry *e )
 392 {
 393         struct bdb_info *bdb = (struct bdb_info *) op->o_bd->be_private;
 394
 395         if ( slapMode & SLAP_TOOL_QUICK ) {
 396                 IndexRec *ir;
 397                 int i, rc;
 398                 Attribute *a;
 399
 400                 ir = bdb_tool_index_rec;
 401                 memset(ir, 0, bdb->bi_nattrs * sizeof( IndexRec ));
 402
 403                 for ( a = e->e_attrs; a != NULL; a = a->a_next ) {
 404                         rc = bdb_index_recset( bdb, a, a->a_desc->ad_type,
 405                                 &a->a_desc->ad_tags, ir );
 406                         if ( rc )
 407                                 return rc;
 408                 }
 409                 bdb_tool_ix_id = e->e_id;
 410                 bdb_tool_ix_op = op;
 411                 ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex );
 412                 for ( i=1; i<slap_tool_thread_max; i++ )
 413                         bdb_tool_index_threads[i] = LDAP_BUSY;
 414                 ldap_pvt_thread_cond_broadcast( &bdb_tool_index_cond );
 415                 ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex );
 416                 rc = bdb_index_recrun( op, bdb, ir, e->e_id, 0 );
 417                 if ( rc )
 418                         return rc;
 419                 for ( i=1; i<slap_tool_thread_max; i++ ) {
 420                         if ( bdb_tool_index_threads[i] == LDAP_BUSY ) {
 421                                 ldap_pvt_thread_yield();
 422                                 i--;
 423                                 continue;
 424                         }
 425                         if ( bdb_tool_index_threads[i] )
 426                                 return bdb_tool_index_threads[i];
 427                 }
 428                 return 0;
 429         } else {
 430                 return bdb_index_entry_add( op, txn, e );
 431         }
 432 }
 433
 434 ID bdb_tool_entry_put(
 435         BackendDB *be,
 436         Entry *e,
 437         struct berval *text )
 438 {
 439         int rc;
 440         struct bdb_info *bdb = (struct bdb_info *) be->be_private;
 441         DB_TXN *tid = NULL;
 442         Operation op = {0};
 443         Opheader ohdr = {0};
 444
 445         assert( be != NULL );
 446         assert( slapMode & SLAP_TOOL_MODE );
 447
 448         assert( text != NULL );
 449         assert( text->bv_val != NULL );
 450         assert( text->bv_val[0] == '\0' );      /* overconservative? */
 451
 452         Debug( LDAP_DEBUG_TRACE, "=> " LDAP_XSTRING(bdb_tool_entry_put)
 453                 "( %ld, \"%s\" )\n", (long) e->e_id, e->e_dn, 0 );
 454
 455         if (! (slapMode & SLAP_TOOL_QUICK)) {
 456         rc = TXN_BEGIN( bdb->bi_dbenv, NULL, &tid,
 457                 bdb->bi_db_opflags );
 458         if( rc != 0 ) {
 459                 snprintf( text->bv_val, text->bv_len,
 460                         "txn_begin failed: %s (%d)",
 461                         db_strerror(rc), rc );
 462                 Debug( LDAP_DEBUG_ANY,
 463                         "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
 464                          text->bv_val, 0, 0 );
 465                 return NOID;
 466         }
 467         }
 468
 469         op.o_hdr = &ohdr;
 470         op.o_bd = be;
 471         op.o_tmpmemctx = NULL;
 472         op.o_tmpmfuncs = &ch_mfuncs;
 473
 474         /* add dn2id indices */
 475         rc = bdb_tool_next_id( &op, tid, e, text, 0 );
 476         if( rc != 0 ) {
 477                 goto done;
 478         }
 479
 480         /* id2entry index */
 481         rc = bdb_id2entry_add( be, tid, e );
 482         if( rc != 0 ) {
 483                 snprintf( text->bv_val, text->bv_len,
 484                                 "id2entry_add failed: %s (%d)",
 485                                 db_strerror(rc), rc );
 486                 Debug( LDAP_DEBUG_ANY,
 487                         "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
 488                         text->bv_val, 0, 0 );
 489                 goto done;
 490         }
 491
 492         if ( !bdb->bi_linear_index )
 493                 rc = bdb_tool_index_add( &op, tid, e );
 494         if( rc != 0 ) {
 495                 snprintf( text->bv_val, text->bv_len,
 496                                 "index_entry_add failed: %s (%d)",
 497                                 db_strerror(rc), rc );
 498                 Debug( LDAP_DEBUG_ANY,
 499                         "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
 500                         text->bv_val, 0, 0 );
 501                 goto done;
 502         }
 503
 504 done:
 505         if( rc == 0 ) {
 506                 if ( !( slapMode & SLAP_TOOL_QUICK )) {
 507                 rc = TXN_COMMIT( tid, 0 );
 508                 if( rc != 0 ) {
 509                         snprintf( text->bv_val, text->bv_len,
 510                                         "txn_commit failed: %s (%d)",
 511                                         db_strerror(rc), rc );
 512                         Debug( LDAP_DEBUG_ANY,
 513                                 "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
 514                                 text->bv_val, 0, 0 );
 515                         e->e_id = NOID;
 516                 }
 517                 }
 518
 519         } else {
 520                 if ( !( slapMode & SLAP_TOOL_QUICK )) {
 521                 TXN_ABORT( tid );
 522                 snprintf( text->bv_val, text->bv_len,
 523                         "txn_aborted! %s (%d)",
 524                         db_strerror(rc), rc );
 525                 Debug( LDAP_DEBUG_ANY,
 526                         "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n",
 527                         text->bv_val, 0, 0 );
 528                 }
 529                 e->e_id = NOID;
 530         }
 531
 532         return e->e_id;
 533 }
 534
 535 int bdb_tool_entry_reindex(
 536         BackendDB *be,
 537         ID id )
 538 {
 539         struct bdb_info *bi = (struct bdb_info *) be->be_private;
 540         int rc;
 541         Entry *e;
 542         DB_TXN *tid = NULL;
 543         Operation op = {0};
 544         Opheader ohdr = {0};
 545
 546         Debug( LDAP_DEBUG_ARGS,
 547                 "=> " LDAP_XSTRING(bdb_tool_entry_reindex) "( %ld )\n",
 548                 (long) id, 0, 0 );
 549
 550         /* No indexes configured, nothing to do. Could return an
 551          * error here to shortcut things.
 552          */
 553         if (!bi->bi_attrs) {
 554                 return 0;
 555         }
 556
 557         /* Get the first attribute to index */
 558         if (bi->bi_linear_index && !index_nattrs) {
 559                 index_nattrs = bi->bi_nattrs - 1;
 560                 bi->bi_nattrs = 1;
 561         }
 562
 563         e = bdb_tool_entry_get( be, id );
 564
 565         if( e == NULL ) {
 566                 Debug( LDAP_DEBUG_ANY,
 567                         LDAP_XSTRING(bdb_tool_entry_reindex)
 568                         ": could not locate id=%ld\n",
 569                         (long) id, 0, 0 );
 570                 return -1;
 571         }
 572
 573         if (! (slapMode & SLAP_TOOL_QUICK)) {
 574         rc = TXN_BEGIN( bi->bi_dbenv, NULL, &tid, bi->bi_db_opflags );
 575         if( rc != 0 ) {
 576                 Debug( LDAP_DEBUG_ANY,
 577                         "=> " LDAP_XSTRING(bdb_tool_entry_reindex) ": "
 578                         "txn_begin failed: %s (%d)\n",
 579                         db_strerror(rc), rc, 0 );
 580                 goto done;
 581         }
 582         }
 583
 584         /*
 585          * just (re)add them for now
 586          * assume that some other routine (not yet implemented)
 587          * will zap index databases
 588          *
 589          */
 590
 591         Debug( LDAP_DEBUG_TRACE,
 592                 "=> " LDAP_XSTRING(bdb_tool_entry_reindex) "( %ld, \"%s\" )\n",
 593                 (long) id, e->e_dn, 0 );
 594
 595         op.o_hdr = &ohdr;
 596         op.o_bd = be;
 597         op.o_tmpmemctx = NULL;
 598         op.o_tmpmfuncs = &ch_mfuncs;
 599
 600         rc = bdb_tool_index_add( &op, tid, e );
 601
 602 done:
 603         if( rc == 0 ) {
 604                 if (! (slapMode & SLAP_TOOL_QUICK)) {
 605                 rc = TXN_COMMIT( tid, 0 );
 606                 if( rc != 0 ) {
 607                         Debug( LDAP_DEBUG_ANY,
 608                                 "=> " LDAP_XSTRING(bdb_tool_entry_reindex)
 609                                 ": txn_commit failed: %s (%d)\n",
 610                                 db_strerror(rc), rc, 0 );
 611                         e->e_id = NOID;
 612                 }
 613                 }
 614
 615         } else {
 616                 if (! (slapMode & SLAP_TOOL_QUICK)) {
 617                 TXN_ABORT( tid );
 618                 Debug( LDAP_DEBUG_ANY,
 619                         "=> " LDAP_XSTRING(bdb_tool_entry_reindex)
 620                         ": txn_aborted! %s (%d)\n",
 621                         db_strerror(rc), rc, 0 );
 622                 }
 623                 e->e_id = NOID;
 624         }
 625         bdb_entry_release( &op, e, 0 );
 626
 627         return rc;
 628 }
 629
 630 ID bdb_tool_entry_modify(
 631         BackendDB *be,
 632         Entry *e,
 633         struct berval *text )
 634 {
 635         int rc;
 636         struct bdb_info *bdb = (struct bdb_info *) be->be_private;
 637         DB_TXN *tid = NULL;
 638         Operation op = {0};
 639         Opheader ohdr = {0};
 640
 641         assert( be != NULL );
 642         assert( slapMode & SLAP_TOOL_MODE );
 643
 644         assert( text != NULL );
 645         assert( text->bv_val != NULL );
 646         assert( text->bv_val[0] == '\0' );      /* overconservative? */
 647
 648         assert ( e->e_id != NOID );
 649
 650         Debug( LDAP_DEBUG_TRACE,
 651                 "=> " LDAP_XSTRING(bdb_tool_entry_modify) "( %ld, \"%s\" )\n",
 652                 (long) e->e_id, e->e_dn, 0 );
 653
 654         if (! (slapMode & SLAP_TOOL_QUICK)) {
 655         rc = TXN_BEGIN( bdb->bi_dbenv, NULL, &tid,
 656                 bdb->bi_db_opflags );
 657         if( rc != 0 ) {
 658                 snprintf( text->bv_val, text->bv_len,
 659                         "txn_begin failed: %s (%d)",
 660                         db_strerror(rc), rc );
 661                 Debug( LDAP_DEBUG_ANY,
 662                         "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n",
 663                          text->bv_val, 0, 0 );
 664                 return NOID;
 665         }
 666         }
 667
 668         op.o_hdr = &ohdr;
 669         op.o_bd = be;
 670         op.o_tmpmemctx = NULL;
 671         op.o_tmpmfuncs = &ch_mfuncs;
 672
 673         /* id2entry index */
 674         rc = bdb_id2entry_update( be, tid, e );
 675         if( rc != 0 ) {
 676                 snprintf( text->bv_val, text->bv_len,
 677                                 "id2entry_add failed: %s (%d)",
 678                                 db_strerror(rc), rc );
 679                 Debug( LDAP_DEBUG_ANY,
 680                         "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n",
 681                         text->bv_val, 0, 0 );
 682                 goto done;
 683         }
 684
 685 #if 0
 686         /* FIXME: this is bogus, we don't have the old values to delete
 687          * from the index because the given entry has already been modified.
 688          */
 689         rc = bdb_index_entry_del( &op, tid, e );
 690         if( rc != 0 ) {
 691                 snprintf( text->bv_val, text->bv_len,
 692                                 "index_entry_del failed: %s (%d)",
 693                                 db_strerror(rc), rc );
 694                 Debug( LDAP_DEBUG_ANY,
 695                         "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n",
 696                         text->bv_val, 0, 0 );
 697                 goto done;
 698         }
 699 #endif
 700
 701         rc = bdb_index_entry_add( &op, tid, e );
 702         if( rc != 0 ) {
 703                 snprintf( text->bv_val, text->bv_len,
 704                                 "index_entry_add failed: %s (%d)",
 705                                 db_strerror(rc), rc );
 706                 Debug( LDAP_DEBUG_ANY,
 707                         "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n",
 708                         text->bv_val, 0, 0 );
 709                 goto done;
 710         }
 711
 712 done:
 713         if( rc == 0 ) {
 714                 if (! (slapMode & SLAP_TOOL_QUICK)) {
 715                 rc = TXN_COMMIT( tid, 0 );
 716                 if( rc != 0 ) {
 717                         snprintf( text->bv_val, text->bv_len,
 718                                         "txn_commit failed: %s (%d)",
 719                                         db_strerror(rc), rc );
 720                         Debug( LDAP_DEBUG_ANY,
 721                                 "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": "
 722                                 "%s\n", text->bv_val, 0, 0 );
 723                         e->e_id = NOID;
 724                 }
 725                 }
 726
 727         } else {
 728                 if (! (slapMode & SLAP_TOOL_QUICK)) {
 729                 TXN_ABORT( tid );
 730                 snprintf( text->bv_val, text->bv_len,
 731                         "txn_aborted! %s (%d)",
 732                         db_strerror(rc), rc );
 733                 Debug( LDAP_DEBUG_ANY,
 734                         "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n",
 735                         text->bv_val, 0, 0 );
 736                 }
 737                 e->e_id = NOID;
 738         }
 739
 740         return e->e_id;
 741 }
 742
 743 #ifdef BDB_TOOL_IDL_CACHING
 744 static int
 745 bdb_tool_idl_cmp( const void *v1, const void *v2 )
 746 {
 747         const bdb_tool_idl_cache *c1 = v1, *c2 = v2;
 748         int rc;
 749
 750         if (( rc = c1->kstr.bv_len - c2->kstr.bv_len )) return rc;
 751         return memcmp( c1->kstr.bv_val, c2->kstr.bv_val, c1->kstr.bv_len );
 752 }
 753
 754 static int
 755 bdb_tool_idl_flush_one( void *v1, void *arg )
 756 {
 757         bdb_tool_idl_cache *ic = v1;
 758         DB *db = arg;
 759         struct bdb_info *bdb = bdb_tool_info;
 760         bdb_tool_idl_cache_entry *ice;
 761         DBC *curs;
 762         DBT key, data;
 763         int i, rc;
 764         ID id, nid;
 765
 766         /* Freshly allocated, ignore it */
 767         if ( !ic->head && ic->count <= BDB_IDL_DB_SIZE ) {
 768                 return 0;
 769         }
 770
 771         rc = db->cursor( db, NULL, &curs, 0 );
 772         if ( rc )
 773                 return -1;
 774
 775         DBTzero( &key );
 776         DBTzero( &data );
 777
 778         bv2DBT( &ic->kstr, &key );
 779
 780         data.size = data.ulen = sizeof( ID );
 781         data.flags = DB_DBT_USERMEM;
 782         data.data = &nid;
 783
 784         rc = curs->c_get( curs, &key, &data, DB_SET );
 785         /* If key already exists and we're writing a range... */
 786         if ( rc == 0 && ic->count > BDB_IDL_DB_SIZE ) {
 787                 /* If it's not currently a range, must delete old info */
 788                 if ( nid ) {
 789                         /* Skip lo */
 790                         while ( curs->c_get( curs, &key, &data, DB_NEXT_DUP ) == 0 )
 791                                 curs->c_del( curs, 0 );
 792
 793                         nid = 0;
 794                         /* Store range marker */
 795                         curs->c_put( curs, &key, &data, DB_KEYFIRST );
 796                 } else {
 797
 798                         /* Skip lo */
 799                         rc = curs->c_get( curs, &key, &data, DB_NEXT_DUP );
 800
 801                         /* Get hi */
 802                         rc = curs->c_get( curs, &key, &data, DB_NEXT_DUP );
 803
 804                         /* Delete hi */
 805                         curs->c_del( curs, 0 );
 806                 }
 807                 BDB_ID2DISK( ic->last, &nid );
 808                 curs->c_put( curs, &key, &data, DB_KEYLAST );
 809                 rc = 0;
 810         } else if ( rc && rc != DB_NOTFOUND ) {
 811                 rc = -1;
 812         } else if ( ic->count > BDB_IDL_DB_SIZE ) {
 813                 /* range, didn't exist before */
 814                 nid = 0;
 815                 rc = curs->c_put( curs, &key, &data, DB_KEYLAST );
 816                 if ( rc == 0 ) {
 817                         BDB_ID2DISK( ic->first, &nid );
 818                         rc = curs->c_put( curs, &key, &data, DB_KEYLAST );
 819                         if ( rc == 0 ) {
 820                                 BDB_ID2DISK( ic->last, &nid );
 821                                 rc = curs->c_put( curs, &key, &data, DB_KEYLAST );
 822                         }
 823                 }
 824                 if ( rc ) {
 825                         rc = -1;
 826                 }
 827         } else {
 828                 int n;
 829
 830                 /* Just a normal write */
 831                 rc = 0;
 832                 for ( ice = ic->head, n=0; ice; ice = ice->next, n++ ) {
 833                         int end;
 834                         if ( ice->next ) {
 835                                 end = IDBLOCK;
 836                         } else {
 837                                 end = ic->count & (IDBLOCK-1);
 838                                 if ( !end )
 839                                         end = IDBLOCK;
 840                         }
 841                         for ( i=0; i<end; i++ ) {
 842                                 if ( !ice->ids[i] ) continue;
 843                                 BDB_ID2DISK( ice->ids[i], &nid );
 844                                 rc = curs->c_put( curs, &key, &data, DB_NODUPDATA );
 845                                 if ( rc ) {
 846                                         if ( rc == DB_KEYEXIST ) {
 847                                                 rc = 0;
 848                                                 continue;
 849                                         }
 850                                         rc = -1;
 851                                         break;
 852                                 }
 853                         }
 854                         if ( rc ) {
 855                                 rc = -1;
 856                                 break;
 857                         }
 858                 }
 859                 if ( ic->head ) {
 860                         ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
 861                         ic->tail->next = bdb_tool_idl_free_list;
 862                         bdb_tool_idl_free_list = ic->head;
 863                         bdb->bi_idl_cache_size -= n;
 864                         ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
 865                 }
 866         }
 867         if ( ic != db->app_private ) {
 868                 ch_free( ic );
 869         } else {
 870                 ic->head = ic->tail = NULL;
 871         }
 872         curs->c_close( curs );
 873         return rc;
 874 }
 875
 876 static int
 877 bdb_tool_idl_flush_db( DB *db, bdb_tool_idl_cache *ic )
 878 {
 879         Avlnode *root = db->app_private;
 880         int rc;
 881
 882         db->app_private = ic;
 883         rc = avl_apply( root, bdb_tool_idl_flush_one, db, -1, AVL_INORDER );
 884         avl_free( root, NULL );
 885         db->app_private = NULL;
 886         if ( rc != -1 )
 887                 rc = 0;
 888         return rc;
 889 }
 890
 891 static int
 892 bdb_tool_idl_flush( BackendDB *be )
 893 {
 894         struct bdb_info *bdb = (struct bdb_info *) be->be_private;
 895         DB *db;
 896         Avlnode *root;
 897         int i, rc = 0;
 898
 899         for ( i=BDB_NDB; i < bdb->bi_ndatabases; i++ ) {
 900                 db = bdb->bi_databases[i]->bdi_db;
 901                 if ( !db->app_private ) continue;
 902                 rc = bdb_tool_idl_flush_db( db, NULL );
 903                 if ( rc )
 904                         break;
 905         }
 906         if ( !rc ) {
 907                 bdb->bi_idl_cache_size = 0;
 908         }
 909         return rc;
 910 }
 911
 912 int bdb_tool_idl_add(
 913         BackendDB *be,
 914         DB *db,
 915         DB_TXN *txn,
 916         DBT *key,
 917         ID id )
 918 {
 919         struct bdb_info *bdb = (struct bdb_info *) be->be_private;
 920         bdb_tool_idl_cache *ic, itmp;
 921         bdb_tool_idl_cache_entry *ice;
 922         int rc;
 923
 924         if ( !bdb->bi_idl_cache_max_size )
 925                 return bdb_idl_insert_key( be, db, txn, key, id );
 926
 927         DBT2bv( key, &itmp.kstr );
 928
 929         ic = avl_find( (Avlnode *)db->app_private, &itmp, bdb_tool_idl_cmp );
 930
 931         /* No entry yet, create one */
 932         if ( !ic ) {
 933                 DBC *curs;
 934                 DBT data;
 935                 ID nid;
 936                 int rc;
 937
 938                 ic = ch_malloc( sizeof( bdb_tool_idl_cache ) + itmp.kstr.bv_len );
 939                 ic->kstr.bv_len = itmp.kstr.bv_len;
 940                 ic->kstr.bv_val = (char *)(ic+1);
 941                 AC_MEMCPY( ic->kstr.bv_val, itmp.kstr.bv_val, ic->kstr.bv_len );
 942                 ic->head = ic->tail = NULL;
 943                 ic->last = 0;
 944                 ic->count = 0;
 945                 avl_insert( (Avlnode **)&db->app_private, ic, bdb_tool_idl_cmp,
 946                         avl_dup_error );
 947
 948                 /* load existing key count here */
 949                 rc = db->cursor( db, NULL, &curs, 0 );
 950                 if ( rc ) return rc;
 951
 952                 data.ulen = sizeof( ID );
 953                 data.flags = DB_DBT_USERMEM;
 954                 data.data = &nid;
 955                 rc = curs->c_get( curs, key, &data, DB_SET );
 956                 if ( rc == 0 ) {
 957                         if ( nid == 0 ) {
 958                                 ic->count = BDB_IDL_DB_SIZE+1;
 959                         } else {
 960                                 db_recno_t count;
 961
 962                                 curs->c_count( curs, &count, 0 );
 963                                 ic->count = count;
 964                                 BDB_DISK2ID( &nid, &ic->first );
 965                         }
 966                 }
 967                 curs->c_close( curs );
 968         }
 969         /* are we a range already? */
 970         if ( ic->count > BDB_IDL_DB_SIZE ) {
 971                 ic->last = id;
 972                 return 0;
 973         /* Are we at the limit, and converting to a range? */
 974         } else if ( ic->count == BDB_IDL_DB_SIZE ) {
 975                 int n;
 976                 for ( ice = ic->head, n=0; ice; ice = ice->next, n++ )
 977                         /* counting */ ;
 978                 if ( n ) {
 979                         ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
 980                         ic->tail->next = bdb_tool_idl_free_list;
 981                         bdb_tool_idl_free_list = ic->head;
 982                         bdb->bi_idl_cache_size -= n;
 983                         ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
 984                 }
 985                 ic->head = ic->tail = NULL;
 986                 ic->last = id;
 987                 ic->count++;
 988                 return 0;
 989         }
 990         /* No free block, create that too */
 991         if ( !ic->tail || ( ic->count & (IDBLOCK-1)) == 0) {
 992                 ice = NULL;
 993                 ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
 994                 if ( bdb->bi_idl_cache_size >= bdb->bi_idl_cache_max_size ) {
 995                         ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
 996                         rc = bdb_tool_idl_flush_db( db, ic );
 997                         if ( rc )
 998                                 return rc;
 999                         avl_insert( (Avlnode **)&db->app_private, ic, bdb_tool_idl_cmp,
1000                                 avl_dup_error );
1001                         ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
1002                 }
1003                 bdb->bi_idl_cache_size++;
1004                 if ( bdb_tool_idl_free_list ) {
1005                         ice = bdb_tool_idl_free_list;
1006                         bdb_tool_idl_free_list = ice->next;
1007                 }
1008                 ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
1009                 if ( !ice ) {
1010                         ice = ch_malloc( sizeof( bdb_tool_idl_cache_entry ));
1011                 }
1012                 memset( ice, 0, sizeof( *ice ));
1013                 if ( !ic->head ) {
1014                         ic->head = ice;
1015                 } else {
1016                         ic->tail->next = ice;
1017                 }
1018                 ic->tail = ice;
1019                 if ( !ic->count )
1020                         ic->first = id;
1021         }
1022         ice = ic->tail;
1023         ice->ids[ ic->count & (IDBLOCK-1) ] = id;
1024         ic->count++;
1025
1026         return 0;
1027 }
1028 #endif
1029
1030 static void *
1031 bdb_tool_index_task( void *ctx, void *ptr )
1032 {
1033         int base = *(int *)ptr;
1034
1035         free( ptr );
1036         while ( 1 ) {
1037                 ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex );
1038                 ldap_pvt_thread_cond_wait( &bdb_tool_index_cond,
1039                         &bdb_tool_index_mutex );
1040                 ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex );
1041                 if ( slapd_shutdown )
1042                         break;
1043
1044                 bdb_tool_index_threads[base] = bdb_index_recrun( bdb_tool_ix_op,
1045                         bdb_tool_info, bdb_tool_index_rec, bdb_tool_ix_id, base );
1046         }
1047
1048         return NULL;
1049 }