git.sur5r.net Git - openldap/blob - servers/slapd/back-mdb/tools.c

   1 /* tools.c - tools for slap tools */
   2 /* $OpenLDAP$ */
   3 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
   4  *
   5  * Copyright 2011-2018 The OpenLDAP Foundation.
   6  * All rights reserved.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted only as authorized by the OpenLDAP
  10  * Public License.
  11  *
  12  * A copy of this license is available in the file LICENSE in the
  13  * top-level directory of the distribution or, alternatively, at
  14  * <http://www.OpenLDAP.org/license.html>.
  15  */
  16
  17 #include "portable.h"
  18
  19 #include <stdio.h>
  20 #include <ac/string.h>
  21 #include <ac/errno.h>
  22
  23 #define AVL_INTERNAL
  24 #include "back-mdb.h"
  25 #include "idl.h"
  26
  27 #ifdef MDB_TOOL_IDL_CACHING
  28 static int mdb_tool_idl_flush( BackendDB *be, MDB_txn *txn );
  29
  30 #define IDBLOCK 1024
  31
  32 typedef struct mdb_tool_idl_cache_entry {
  33         struct mdb_tool_idl_cache_entry *next;
  34         ID ids[IDBLOCK];
  35 } mdb_tool_idl_cache_entry;
  36
  37 typedef struct mdb_tool_idl_cache {
  38         struct berval kstr;
  39         mdb_tool_idl_cache_entry *head, *tail;
  40         ID first, last;
  41         int count;
  42         short offset;
  43         short flags;
  44 } mdb_tool_idl_cache;
  45 #define WAS_FOUND       0x01
  46 #define WAS_RANGE       0x02
  47
  48 #define MDB_TOOL_IDL_FLUSH(be, txn)     mdb_tool_idl_flush(be, txn)
  49 #else
  50 #define MDB_TOOL_IDL_FLUSH(be, txn)
  51 #endif /* MDB_TOOL_IDL_CACHING */
  52
  53 MDB_txn *mdb_tool_txn = NULL;
  54
  55 static MDB_txn *txi = NULL;
  56 static MDB_cursor *cursor = NULL, *idcursor = NULL;
  57 static MDB_cursor *mcp = NULL, *mcd = NULL;
  58 static MDB_val key, data;
  59 static ID previd = NOID;
  60
  61 typedef struct dn_id {
  62         ID id;
  63         struct berval dn;
  64 } dn_id;
  65
  66 #define HOLE_SIZE       4096
  67 static dn_id hbuf[HOLE_SIZE], *holes = hbuf;
  68 static unsigned nhmax = HOLE_SIZE;
  69 static unsigned nholes;
  70
  71 static struct berval    *tool_base;
  72 static int              tool_scope;
  73 static Filter           *tool_filter;
  74 static Entry            *tool_next_entry;
  75
  76 static ID mdb_tool_ix_id;
  77 static BackendDB *mdb_tool_ix_be;
  78 static MDB_txn *mdb_tool_ix_txn;
  79 static int mdb_tool_index_tcount, mdb_tool_threads;
  80 static IndexRec *mdb_tool_index_rec;
  81 static AttrIxInfo **mdb_tool_axinfo;
  82 static struct mdb_info *mdb_tool_info;
  83 static ldap_pvt_thread_mutex_t mdb_tool_index_mutex;
  84 static ldap_pvt_thread_cond_t mdb_tool_index_cond_main;
  85 static ldap_pvt_thread_cond_t mdb_tool_index_cond_work;
  86 static void * mdb_tool_index_task( void *ctx, void *ptr );
  87
  88 static int      mdb_writes, mdb_writes_per_commit;
  89
  90 /* Number of ops per commit in Quick mode.
  91  * Batching speeds writes overall, but too large a
  92  * batch will fail with MDB_TXN_FULL.
  93  */
  94 #ifndef MDB_WRITES_PER_COMMIT
  95 #define MDB_WRITES_PER_COMMIT   500
  96 #endif
  97
  98 static int
  99 mdb_tool_entry_get_int( BackendDB *be, ID id, Entry **ep );
 100
 101 int mdb_tool_entry_open(
 102         BackendDB *be, int mode )
 103 {
 104         /* In Quick mode, commit once per 500 entries */
 105         mdb_writes = 0;
 106         if ( slapMode & SLAP_TOOL_QUICK )
 107                 mdb_writes_per_commit = MDB_WRITES_PER_COMMIT;
 108         else
 109                 mdb_writes_per_commit = 1;
 110
 111 #ifdef MDB_TOOL_IDL_CACHING                     /* threaded indexing has no performance advantage */
 112         /* Set up for threaded slapindex */
 113         if (( slapMode & (SLAP_TOOL_QUICK|SLAP_TOOL_READONLY)) == SLAP_TOOL_QUICK ) {
 114                 if ( !mdb_tool_info ) {
 115                         struct mdb_info *mdb = (struct mdb_info *) be->be_private;
 116                         ldap_pvt_thread_mutex_init( &mdb_tool_index_mutex );
 117                         ldap_pvt_thread_cond_init( &mdb_tool_index_cond_main );
 118                         ldap_pvt_thread_cond_init( &mdb_tool_index_cond_work );
 119                         if ( mdb->mi_nattrs ) {
 120                                 int i;
 121                                 mdb_tool_threads = slap_tool_thread_max - 1;
 122                                 if ( mdb_tool_threads > 1 ) {
 123                                         mdb_tool_index_rec = ch_calloc( mdb->mi_nattrs, sizeof( IndexRec ));
 124                                         mdb_tool_axinfo = ch_calloc( mdb_tool_threads, sizeof( AttrIxInfo* ) +
 125                                                 sizeof( AttrIxInfo ));
 126                                         mdb_tool_axinfo[0] = (AttrIxInfo *)(mdb_tool_axinfo + mdb_tool_threads);
 127                                         for (i=1; i<mdb_tool_threads; i++)
 128                                                 mdb_tool_axinfo[i] = mdb_tool_axinfo[i-1]+1;
 129                                         mdb_tool_index_tcount = mdb_tool_threads - 1;
 130                                         mdb_tool_ix_be = be;
 131                                         for (i=1; i<mdb_tool_threads; i++) {
 132                                                 int *ptr = ch_malloc( sizeof( int ));
 133                                                 *ptr = i;
 134                                                 ldap_pvt_thread_pool_submit( &connection_pool,
 135                                                         mdb_tool_index_task, ptr );
 136                                         }
 137                                         mdb_tool_info = mdb;
 138                                 }
 139                         }
 140                 }
 141         }
 142 #endif
 143
 144         return 0;
 145 }
 146
 147 int mdb_tool_entry_close(
 148         BackendDB *be )
 149 {
 150 #ifdef MDB_TOOL_IDL_CACHING
 151         if ( mdb_tool_info ) {
 152                 int i;
 153                 slapd_shutdown = 1;
 154                 ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex );
 155
 156                 /* There might still be some threads starting */
 157                 while ( mdb_tool_index_tcount > 0 ) {
 158                         ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main,
 159                                         &mdb_tool_index_mutex );
 160                 }
 161
 162                 mdb_tool_index_tcount = mdb_tool_threads - 1;
 163                 ldap_pvt_thread_cond_broadcast( &mdb_tool_index_cond_work );
 164
 165                 /* Make sure all threads are stopped */
 166                 while ( mdb_tool_index_tcount > 0 ) {
 167                         ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main,
 168                                 &mdb_tool_index_mutex );
 169                 }
 170                 ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex );
 171
 172                 mdb_tool_info = NULL;
 173                 slapd_shutdown = 0;
 174                 ch_free( mdb_tool_index_rec );
 175                 mdb_tool_index_tcount = mdb_tool_threads - 1;
 176                 if (mdb_tool_txn)
 177                         MDB_TOOL_IDL_FLUSH( be, mdb_tool_txn );
 178                 for (i=0; i<mdb_tool_threads; i++) {
 179                         mdb_tool_idl_cache *ic;
 180                         mdb_tool_idl_cache_entry *ice;
 181                         while ((ic = mdb_tool_axinfo[i]->ai_clist)) {
 182                                 mdb_tool_axinfo[i]->ai_clist = ic->head;
 183                                 free(ic);
 184                         }
 185                         while ((ice = mdb_tool_axinfo[i]->ai_flist)) {
 186                                 mdb_tool_axinfo[i]->ai_flist = ice->next;
 187                                 free(ice);
 188                         }
 189                 }
 190         }
 191 #endif
 192
 193         if( idcursor ) {
 194                 mdb_cursor_close( idcursor );
 195                 idcursor = NULL;
 196         }
 197         if( cursor ) {
 198                 mdb_cursor_close( cursor );
 199                 cursor = NULL;
 200         }
 201         {
 202                 struct mdb_info *mdb = be->be_private;
 203                 if ( mdb ) {
 204                         int i;
 205                         for (i=0; i<mdb->mi_nattrs; i++)
 206                                 mdb->mi_attrs[i]->ai_cursor = NULL;
 207                 }
 208         }
 209         if( mdb_tool_txn ) {
 210                 int rc;
 211                 if (( rc = mdb_txn_commit( mdb_tool_txn ))) {
 212                         Debug( LDAP_DEBUG_ANY,
 213                                 LDAP_XSTRING(mdb_tool_entry_close) ": database %s: "
 214                                 "txn_commit failed: %s (%d)\n",
 215                                 be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
 216                         return -1;
 217                 }
 218                 mdb_tool_txn = NULL;
 219         }
 220
 221         if( nholes ) {
 222                 unsigned i;
 223                 fprintf( stderr, "Error, entries missing!\n");
 224                 for (i=0; i<nholes; i++) {
 225                         fprintf(stderr, "  entry %ld: %s\n",
 226                                 holes[i].id, holes[i].dn.bv_val);
 227                 }
 228                 nholes = 0;
 229                 return -1;
 230         }
 231
 232         return 0;
 233 }
 234
 235 ID
 236 mdb_tool_entry_first_x(
 237         BackendDB *be,
 238         struct berval *base,
 239         int scope,
 240         Filter *f )
 241 {
 242         tool_base = base;
 243         tool_scope = scope;
 244         tool_filter = f;
 245
 246         return mdb_tool_entry_next( be );
 247 }
 248
 249 ID mdb_tool_entry_next(
 250         BackendDB *be )
 251 {
 252         int rc;
 253         ID id;
 254         struct mdb_info *mdb;
 255
 256         assert( be != NULL );
 257         assert( slapMode & SLAP_TOOL_MODE );
 258
 259         mdb = (struct mdb_info *) be->be_private;
 260         assert( mdb != NULL );
 261
 262         if ( !mdb_tool_txn ) {
 263                 rc = mdb_txn_begin( mdb->mi_dbenv, NULL, MDB_RDONLY, &mdb_tool_txn );
 264                 if ( rc )
 265                         return NOID;
 266                 rc = mdb_cursor_open( mdb_tool_txn, mdb->mi_id2entry, &cursor );
 267                 if ( rc ) {
 268                         mdb_txn_abort( mdb_tool_txn );
 269                         return NOID;
 270                 }
 271         }
 272
 273 next:;
 274         rc = mdb_cursor_get( cursor, &key, &data, MDB_NEXT );
 275
 276         if( rc ) {
 277                 return NOID;
 278         }
 279
 280         previd = *(ID *)key.mv_data;
 281         id = previd;
 282
 283         if ( !data.mv_size )
 284                 goto next;
 285
 286         if ( tool_filter || tool_base ) {
 287                 static Operation op = {0};
 288                 static Opheader ohdr = {0};
 289
 290                 op.o_hdr = &ohdr;
 291                 op.o_bd = be;
 292                 op.o_tmpmemctx = NULL;
 293                 op.o_tmpmfuncs = &ch_mfuncs;
 294
 295                 if ( tool_next_entry ) {
 296                         mdb_entry_release( &op, tool_next_entry, 0 );
 297                         tool_next_entry = NULL;
 298                 }
 299
 300                 rc = mdb_tool_entry_get_int( be, id, &tool_next_entry );
 301                 if ( rc == LDAP_NO_SUCH_OBJECT ) {
 302                         goto next;
 303                 }
 304
 305                 assert( tool_next_entry != NULL );
 306
 307                 if ( tool_filter && test_filter( NULL, tool_next_entry, tool_filter ) != LDAP_COMPARE_TRUE )
 308                 {
 309                         mdb_entry_release( &op, tool_next_entry, 0 );
 310                         tool_next_entry = NULL;
 311                         goto next;
 312                 }
 313         }
 314
 315         return id;
 316 }
 317
 318 ID mdb_tool_dn2id_get(
 319         Backend *be,
 320         struct berval *dn
 321 )
 322 {
 323         struct mdb_info *mdb;
 324         Operation op = {0};
 325         Opheader ohdr = {0};
 326         ID id;
 327         int rc;
 328
 329         if ( BER_BVISEMPTY(dn) )
 330                 return 0;
 331
 332         mdb = (struct mdb_info *) be->be_private;
 333
 334         if ( !mdb_tool_txn ) {
 335                 rc = mdb_txn_begin( mdb->mi_dbenv, NULL, (slapMode & SLAP_TOOL_READONLY) != 0 ?
 336                         MDB_RDONLY : 0, &mdb_tool_txn );
 337                 if ( rc )
 338                         return NOID;
 339         }
 340
 341         op.o_hdr = &ohdr;
 342         op.o_bd = be;
 343         op.o_tmpmemctx = NULL;
 344         op.o_tmpmfuncs = &ch_mfuncs;
 345
 346         rc = mdb_dn2id( &op, mdb_tool_txn, NULL, dn, &id, NULL, NULL, NULL );
 347         if ( rc == MDB_NOTFOUND )
 348                 return NOID;
 349
 350         return id;
 351 }
 352
 353 static int
 354 mdb_tool_entry_get_int( BackendDB *be, ID id, Entry **ep )
 355 {
 356         Operation op = {0};
 357         Opheader ohdr = {0};
 358
 359         Entry *e = NULL;
 360         struct berval dn = BER_BVNULL, ndn = BER_BVNULL;
 361         int rc;
 362
 363         assert( be != NULL );
 364         assert( slapMode & SLAP_TOOL_MODE );
 365
 366         if ( ( tool_filter || tool_base ) && id == previd && tool_next_entry != NULL ) {
 367                 *ep = tool_next_entry;
 368                 tool_next_entry = NULL;
 369                 return LDAP_SUCCESS;
 370         }
 371
 372         if ( id != previd ) {
 373                 key.mv_size = sizeof(ID);
 374                 key.mv_data = &id;
 375                 rc = mdb_cursor_get( cursor, &key, &data, MDB_SET );
 376                 if ( rc ) {
 377                         rc = LDAP_OTHER;
 378                         goto done;
 379                 }
 380         }
 381         if ( !data.mv_size ) {
 382                 rc = LDAP_NO_SUCH_OBJECT;
 383                 goto done;
 384         }
 385
 386         op.o_hdr = &ohdr;
 387         op.o_bd = be;
 388         op.o_tmpmemctx = NULL;
 389         op.o_tmpmfuncs = &ch_mfuncs;
 390         if ( slapMode & SLAP_TOOL_READONLY ) {
 391                 rc = mdb_id2name( &op, mdb_tool_txn, &idcursor, id, &dn, &ndn );
 392                 if ( rc  ) {
 393                         rc = LDAP_OTHER;
 394                         goto done;
 395                 }
 396                 if ( tool_base != NULL ) {
 397                         if ( !dnIsSuffixScope( &ndn, tool_base, tool_scope ) ) {
 398                                 ch_free( dn.bv_val );
 399                                 ch_free( ndn.bv_val );
 400                                 rc = LDAP_NO_SUCH_OBJECT;
 401                                 goto done;
 402                         }
 403                 }
 404         }
 405         rc = mdb_entry_decode( &op, mdb_tool_txn, &data, id, &e );
 406         e->e_id = id;
 407         if ( !BER_BVISNULL( &dn )) {
 408                 e->e_name = dn;
 409                 e->e_nname = ndn;
 410         } else {
 411                 e->e_name.bv_val = NULL;
 412                 e->e_nname.bv_val = NULL;
 413         }
 414
 415 done:
 416         if ( e != NULL ) {
 417                 *ep = e;
 418         }
 419
 420         return rc;
 421 }
 422
 423 Entry*
 424 mdb_tool_entry_get( BackendDB *be, ID id )
 425 {
 426         Entry *e = NULL;
 427         int rc;
 428
 429         if ( !mdb_tool_txn ) {
 430                 struct mdb_info *mdb = (struct mdb_info *) be->be_private;
 431                 rc = mdb_txn_begin( mdb->mi_dbenv, NULL,
 432                         (slapMode & SLAP_TOOL_READONLY) ? MDB_RDONLY : 0, &mdb_tool_txn );
 433                 if ( rc )
 434                         return NULL;
 435         }
 436         if ( !cursor ) {
 437                 struct mdb_info *mdb = (struct mdb_info *) be->be_private;
 438                 rc = mdb_cursor_open( mdb_tool_txn, mdb->mi_id2entry, &cursor );
 439                 if ( rc ) {
 440                         mdb_txn_abort( mdb_tool_txn );
 441                         mdb_tool_txn = NULL;
 442                         return NULL;
 443                 }
 444         }
 445         (void)mdb_tool_entry_get_int( be, id, &e );
 446         return e;
 447 }
 448
 449 static int mdb_tool_next_id(
 450         Operation *op,
 451         MDB_txn *tid,
 452         Entry *e,
 453         struct berval *text,
 454         int hole )
 455 {
 456         struct berval dn = e->e_name;
 457         struct berval ndn = e->e_nname;
 458         struct berval pdn, npdn, nmatched;
 459         ID id, pid = 0;
 460         int rc;
 461
 462         if (ndn.bv_len == 0) {
 463                 e->e_id = 0;
 464                 return 0;
 465         }
 466
 467         rc = mdb_dn2id( op, tid, mcp, &ndn, &id, NULL, NULL, &nmatched );
 468         if ( rc == MDB_NOTFOUND ) {
 469                 if ( !be_issuffix( op->o_bd, &ndn ) ) {
 470                         ID eid = e->e_id;
 471                         dnParent( &ndn, &npdn );
 472                         if ( nmatched.bv_len != npdn.bv_len ) {
 473                                 dnParent( &dn, &pdn );
 474                                 e->e_name = pdn;
 475                                 e->e_nname = npdn;
 476                                 rc = mdb_tool_next_id( op, tid, e, text, 1 );
 477                                 e->e_name = dn;
 478                                 e->e_nname = ndn;
 479                                 if ( rc ) {
 480                                         return rc;
 481                                 }
 482                                 /* If parent didn't exist, it was created just now
 483                                  * and its ID is now in e->e_id. Make sure the current
 484                                  * entry gets added under the new parent ID.
 485                                  */
 486                                 if ( eid != e->e_id ) {
 487                                         pid = e->e_id;
 488                                 }
 489                         } else {
 490                                 pid = id;
 491                         }
 492                 }
 493                 rc = mdb_next_id( op->o_bd, idcursor, &e->e_id );
 494                 if ( rc ) {
 495                         snprintf( text->bv_val, text->bv_len,
 496                                 "next_id failed: %s (%d)",
 497                                 mdb_strerror(rc), rc );
 498                 Debug( LDAP_DEBUG_ANY,
 499                         "=> mdb_tool_next_id: %s\n", text->bv_val, 0, 0 );
 500                         return rc;
 501                 }
 502                 rc = mdb_dn2id_add( op, mcp, mcd, pid, 1, 1, e );
 503                 if ( rc ) {
 504                         snprintf( text->bv_val, text->bv_len,
 505                                 "dn2id_add failed: %s (%d)",
 506                                 mdb_strerror(rc), rc );
 507                         Debug( LDAP_DEBUG_ANY,
 508                                 "=> mdb_tool_next_id: %s\n", text->bv_val, 0, 0 );
 509                 } else if ( hole ) {
 510                         MDB_val key, data;
 511                         if ( nholes == nhmax - 1 ) {
 512                                 if ( holes == hbuf ) {
 513                                         holes = ch_malloc( nhmax * sizeof(dn_id) * 2 );
 514                                         AC_MEMCPY( holes, hbuf, sizeof(hbuf) );
 515                                 } else {
 516                                         holes = ch_realloc( holes, nhmax * sizeof(dn_id) * 2 );
 517                                 }
 518                                 nhmax *= 2;
 519                         }
 520                         ber_dupbv( &holes[nholes].dn, &ndn );
 521                         holes[nholes++].id = e->e_id;
 522                         key.mv_size = sizeof(ID);
 523                         key.mv_data = &e->e_id;
 524                         data.mv_size = 0;
 525                         data.mv_data = NULL;
 526                         rc = mdb_cursor_put( idcursor, &key, &data, MDB_NOOVERWRITE );
 527                         if ( rc == MDB_KEYEXIST )
 528                                 rc = 0;
 529                         if ( rc ) {
 530                                 snprintf( text->bv_val, text->bv_len,
 531                                         "dummy id2entry add failed: %s (%d)",
 532                                         mdb_strerror(rc), rc );
 533                                 Debug( LDAP_DEBUG_ANY,
 534                                         "=> mdb_tool_next_id: %s\n", text->bv_val, 0, 0 );
 535                         }
 536                 }
 537         } else if ( !hole ) {
 538                 unsigned i, j;
 539
 540                 e->e_id = id;
 541
 542                 for ( i=0; i<nholes; i++) {
 543                         if ( holes[i].id == e->e_id ) {
 544                                 free(holes[i].dn.bv_val);
 545                                 for (j=i;j<nholes;j++) holes[j] = holes[j+1];
 546                                 holes[j].id = 0;
 547                                 nholes--;
 548                                 break;
 549                         } else if ( holes[i].id > e->e_id ) {
 550                                 break;
 551                         }
 552                 }
 553         }
 554         return rc;
 555 }
 556
 557 static int
 558 mdb_tool_index_add(
 559         Operation *op,
 560         MDB_txn *txn,
 561         Entry *e )
 562 {
 563         struct mdb_info *mdb = (struct mdb_info *) op->o_bd->be_private;
 564
 565         if ( !mdb->mi_nattrs )
 566                 return 0;
 567
 568         if ( mdb_tool_threads > 1 ) {
 569                 IndexRec *ir;
 570                 int i, rc;
 571                 Attribute *a;
 572
 573                 ir = mdb_tool_index_rec;
 574                 for (i=0; i<mdb->mi_nattrs; i++)
 575                         ir[i].ir_attrs = NULL;
 576
 577                 for ( a = e->e_attrs; a != NULL; a = a->a_next ) {
 578                         rc = mdb_index_recset( mdb, a, a->a_desc->ad_type,
 579                                 &a->a_desc->ad_tags, ir );
 580                         if ( rc )
 581                                 return rc;
 582                 }
 583                 for (i=0; i<mdb->mi_nattrs; i++) {
 584                         if ( !ir[i].ir_ai )
 585                                 break;
 586                         rc = mdb_cursor_open( txn, ir[i].ir_ai->ai_dbi,
 587                                  &ir[i].ir_ai->ai_cursor );
 588                         if ( rc )
 589                                 return rc;
 590                 }
 591                 mdb_tool_ix_id = e->e_id;
 592                 mdb_tool_ix_txn = txn;
 593                 ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex );
 594                 /* Wait for all threads to be ready */
 595                 while ( mdb_tool_index_tcount ) {
 596                         ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main,
 597                                 &mdb_tool_index_mutex );
 598                 }
 599
 600                 for ( i=1; i<mdb_tool_threads; i++ )
 601                         mdb_tool_index_rec[i].ir_i = LDAP_BUSY;
 602                 mdb_tool_index_tcount = mdb_tool_threads - 1;
 603                 ldap_pvt_thread_cond_broadcast( &mdb_tool_index_cond_work );
 604                 ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex );
 605
 606                 return mdb_index_recrun( op, txn, mdb, ir, e->e_id, 0 );
 607         } else
 608         {
 609                 return mdb_index_entry_add( op, txn, e );
 610         }
 611 }
 612
 613 static int
 614 mdb_tool_index_finish()
 615 {
 616         int i, rc = 0;
 617         ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex );
 618         for ( i=1; i<mdb_tool_threads; i++ ) {
 619                 if ( mdb_tool_index_rec[i].ir_i == LDAP_BUSY ) {
 620                         ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main,
 621                                 &mdb_tool_index_mutex );
 622                         i--;
 623                         continue;
 624                 }
 625                 if ( mdb_tool_index_rec[i].ir_i ) {
 626                         rc = mdb_tool_index_rec[i].ir_i;
 627                         break;
 628                 }
 629         }
 630         ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex );
 631         return rc;
 632 }
 633
 634 ID mdb_tool_entry_put(
 635         BackendDB *be,
 636         Entry *e,
 637         struct berval *text )
 638 {
 639         int rc;
 640         struct mdb_info *mdb;
 641         Operation op = {0};
 642         Opheader ohdr = {0};
 643
 644         assert( be != NULL );
 645         assert( slapMode & SLAP_TOOL_MODE );
 646
 647         assert( text != NULL );
 648         assert( text->bv_val != NULL );
 649         assert( text->bv_val[0] == '\0' );      /* overconservative? */
 650
 651         Debug( LDAP_DEBUG_TRACE, "=> " LDAP_XSTRING(mdb_tool_entry_put)
 652                 "( %ld, \"%s\" )\n", (long) e->e_id, e->e_dn, 0 );
 653
 654         mdb = (struct mdb_info *) be->be_private;
 655
 656         if ( !mdb_tool_txn ) {
 657                 rc = mdb_txn_begin( mdb->mi_dbenv, NULL, 0, &mdb_tool_txn );
 658                 if( rc != 0 ) {
 659                         snprintf( text->bv_val, text->bv_len,
 660                                 "txn_begin failed: %s (%d)",
 661                                 mdb_strerror(rc), rc );
 662                         Debug( LDAP_DEBUG_ANY,
 663                                 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
 664                                  text->bv_val, 0, 0 );
 665                         return NOID;
 666                 }
 667                 rc = mdb_cursor_open( mdb_tool_txn, mdb->mi_id2entry, &idcursor );
 668                 if( rc != 0 ) {
 669                         snprintf( text->bv_val, text->bv_len,
 670                                 "cursor_open failed: %s (%d)",
 671                                 mdb_strerror(rc), rc );
 672                         Debug( LDAP_DEBUG_ANY,
 673                                 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
 674                                  text->bv_val, 0, 0 );
 675                         return NOID;
 676                 }
 677                 if ( !mdb->mi_nextid ) {
 678                         ID dummy;
 679                         mdb_next_id( be, idcursor, &dummy );
 680                 }
 681                 rc = mdb_cursor_open( mdb_tool_txn, mdb->mi_dn2id, &mcp );
 682                 if( rc != 0 ) {
 683                         snprintf( text->bv_val, text->bv_len,
 684                                 "cursor_open failed: %s (%d)",
 685                                 mdb_strerror(rc), rc );
 686                         Debug( LDAP_DEBUG_ANY,
 687                                 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
 688                                  text->bv_val, 0, 0 );
 689                         return NOID;
 690                 }
 691                 rc = mdb_cursor_open( mdb_tool_txn, mdb->mi_dn2id, &mcd );
 692                 if( rc != 0 ) {
 693                         snprintf( text->bv_val, text->bv_len,
 694                                 "cursor_open failed: %s (%d)",
 695                                 mdb_strerror(rc), rc );
 696                         Debug( LDAP_DEBUG_ANY,
 697                                 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
 698                                  text->bv_val, 0, 0 );
 699                         return NOID;
 700                 }
 701         }
 702
 703         op.o_hdr = &ohdr;
 704         op.o_bd = be;
 705         op.o_tmpmemctx = NULL;
 706         op.o_tmpmfuncs = &ch_mfuncs;
 707
 708         /* add dn2id indices */
 709         rc = mdb_tool_next_id( &op, mdb_tool_txn, e, text, 0 );
 710         if( rc != 0 ) {
 711                 goto done;
 712         }
 713
 714         if ( mdb_tool_threads > 1 ) {
 715                 LDAP_SLIST_INSERT_HEAD( &op.o_extra, &mdb_tool_axinfo[0]->ai_oe, oe_next );
 716         }
 717         rc = mdb_tool_index_add( &op, mdb_tool_txn, e );
 718         if( rc != 0 ) {
 719                 snprintf( text->bv_val, text->bv_len,
 720                                 "index_entry_add failed: err=%d", rc );
 721                 Debug( LDAP_DEBUG_ANY,
 722                         "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
 723                         text->bv_val, 0, 0 );
 724                 goto done;
 725         }
 726
 727
 728         /* id2entry index */
 729         rc = mdb_id2entry_add( &op, mdb_tool_txn, idcursor, e );
 730         if( rc != 0 ) {
 731                 snprintf( text->bv_val, text->bv_len,
 732                                 "id2entry_add failed: err=%d", rc );
 733                 Debug( LDAP_DEBUG_ANY,
 734                         "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
 735                         text->bv_val, 0, 0 );
 736                 goto done;
 737         }
 738
 739         if( mdb->mi_nattrs && mdb_tool_threads > 1 )
 740                 rc = mdb_tool_index_finish();
 741
 742 done:
 743         if( rc == 0 ) {
 744                 mdb_writes++;
 745                 if ( mdb_writes >= mdb_writes_per_commit ) {
 746                         unsigned i;
 747                         MDB_TOOL_IDL_FLUSH( be, mdb_tool_txn );
 748                         rc = mdb_txn_commit( mdb_tool_txn );
 749                         for ( i=0; i<mdb->mi_nattrs; i++ )
 750                                 mdb->mi_attrs[i]->ai_cursor = NULL;
 751                         mdb_writes = 0;
 752                         mdb_tool_txn = NULL;
 753                         idcursor = NULL;
 754                         if( rc != 0 ) {
 755                                 mdb->mi_numads = 0;
 756                                 snprintf( text->bv_val, text->bv_len,
 757                                                 "txn_commit failed: %s (%d)",
 758                                                 mdb_strerror(rc), rc );
 759                                 Debug( LDAP_DEBUG_ANY,
 760                                         "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
 761                                         text->bv_val, 0, 0 );
 762                                 e->e_id = NOID;
 763                         }
 764                 }
 765
 766         } else {
 767                 unsigned i;
 768                 mdb_txn_abort( mdb_tool_txn );
 769                 mdb_tool_txn = NULL;
 770                 idcursor = NULL;
 771                 for ( i=0; i<mdb->mi_nattrs; i++ )
 772                         mdb->mi_attrs[i]->ai_cursor = NULL;
 773                 mdb_writes = 0;
 774                 snprintf( text->bv_val, text->bv_len,
 775                         "txn_aborted! %s (%d)",
 776                         rc == LDAP_OTHER ? "Internal error" :
 777                         mdb_strerror(rc), rc );
 778                 Debug( LDAP_DEBUG_ANY,
 779                         "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
 780                         text->bv_val, 0, 0 );
 781                 e->e_id = NOID;
 782         }
 783
 784         return e->e_id;
 785 }
 786
 787 static int mdb_dn2id_upgrade( BackendDB *be );
 788
 789 int mdb_tool_entry_reindex(
 790         BackendDB *be,
 791         ID id,
 792         AttributeDescription **adv )
 793 {
 794         struct mdb_info *mi = (struct mdb_info *) be->be_private;
 795         int rc;
 796         Entry *e;
 797         Operation op = {0};
 798         Opheader ohdr = {0};
 799
 800         Debug( LDAP_DEBUG_ARGS,
 801                 "=> " LDAP_XSTRING(mdb_tool_entry_reindex) "( %ld )\n",
 802                 (long) id, 0, 0 );
 803         assert( tool_base == NULL );
 804         assert( tool_filter == NULL );
 805
 806         /* Special: do a dn2id upgrade */
 807         if ( adv && adv[0] == slap_schema.si_ad_entryDN ) {
 808                 /* short-circuit tool_entry_next() */
 809                 mdb_cursor_get( cursor, &key, &data, MDB_LAST );
 810                 return mdb_dn2id_upgrade( be );
 811         }
 812
 813         /* No indexes configured, nothing to do. Could return an
 814          * error here to shortcut things.
 815          */
 816         if (!mi->mi_attrs) {
 817                 return 0;
 818         }
 819
 820         /* Check for explicit list of attrs to index */
 821         if ( adv ) {
 822                 int i, j, n;
 823
 824                 if ( mi->mi_attrs[0]->ai_desc != adv[0] ) {
 825                         /* count */
 826                         for ( n = 0; adv[n]; n++ ) ;
 827
 828                         /* insertion sort */
 829                         for ( i = 0; i < n; i++ ) {
 830                                 AttributeDescription *ad = adv[i];
 831                                 for ( j = i-1; j>=0; j--) {
 832                                         if ( SLAP_PTRCMP( adv[j], ad ) <= 0 ) break;
 833                                         adv[j+1] = adv[j];
 834                                 }
 835                                 adv[j+1] = ad;
 836                         }
 837                 }
 838
 839                 for ( i = 0; adv[i]; i++ ) {
 840                         if ( mi->mi_attrs[i]->ai_desc != adv[i] ) {
 841                                 for ( j = i+1; j < mi->mi_nattrs; j++ ) {
 842                                         if ( mi->mi_attrs[j]->ai_desc == adv[i] ) {
 843                                                 AttrInfo *ai = mi->mi_attrs[i];
 844                                                 mi->mi_attrs[i] = mi->mi_attrs[j];
 845                                                 mi->mi_attrs[j] = ai;
 846                                                 break;
 847                                         }
 848                                 }
 849                                 if ( j == mi->mi_nattrs ) {
 850                                         Debug( LDAP_DEBUG_ANY,
 851                                                 LDAP_XSTRING(mdb_tool_entry_reindex)
 852                                                 ": no index configured for %s\n",
 853                                                 adv[i]->ad_cname.bv_val, 0, 0 );
 854                                         return -1;
 855                                 }
 856                         }
 857                 }
 858                 mi->mi_nattrs = i;
 859         }
 860
 861         e = mdb_tool_entry_get( be, id );
 862
 863         if( e == NULL ) {
 864                 Debug( LDAP_DEBUG_ANY,
 865                         LDAP_XSTRING(mdb_tool_entry_reindex)
 866                         ": could not locate id=%ld\n",
 867                         (long) id, 0, 0 );
 868                 return -1;
 869         }
 870
 871         if ( !txi ) {
 872                 rc = mdb_txn_begin( mi->mi_dbenv, NULL, 0, &txi );
 873                 if( rc != 0 ) {
 874                         Debug( LDAP_DEBUG_ANY,
 875                                 "=> " LDAP_XSTRING(mdb_tool_entry_reindex) ": "
 876                                 "txn_begin failed: %s (%d)\n",
 877                                 mdb_strerror(rc), rc, 0 );
 878                         goto done;
 879                 }
 880         }
 881
 882         if ( slapMode & SLAP_TRUNCATE_MODE ) {
 883                 int i;
 884                 for ( i=0; i < mi->mi_nattrs; i++ ) {
 885                         rc = mdb_drop( txi, mi->mi_attrs[i]->ai_dbi, 0 );
 886                         if ( rc ) {
 887                                 Debug( LDAP_DEBUG_ANY,
 888                                         LDAP_XSTRING(mdb_tool_entry_reindex)
 889                                         ": (Truncate) mdb_drop(%s) failed: %s (%d)\n",
 890                                         mi->mi_attrs[i]->ai_desc->ad_type->sat_cname.bv_val,
 891                                         mdb_strerror(rc), rc );
 892                                 return -1;
 893                         }
 894                 }
 895                 slapMode ^= SLAP_TRUNCATE_MODE;
 896         }
 897
 898         /*
 899          * just (re)add them for now
 900          * Use truncate mode to empty/reset index databases
 901          */
 902
 903         Debug( LDAP_DEBUG_TRACE,
 904                 "=> " LDAP_XSTRING(mdb_tool_entry_reindex) "( %ld )\n",
 905                 (long) id, 0, 0 );
 906
 907         op.o_hdr = &ohdr;
 908         op.o_bd = be;
 909         op.o_tmpmemctx = NULL;
 910         op.o_tmpmfuncs = &ch_mfuncs;
 911
 912         rc = mdb_tool_index_add( &op, txi, e );
 913
 914 done:
 915         if( rc == 0 ) {
 916                 mdb_writes++;
 917                 if ( mdb_writes >= mdb_writes_per_commit ) {
 918                         MDB_val key;
 919                         unsigned i;
 920                         MDB_TOOL_IDL_FLUSH( be, txi );
 921                         rc = mdb_txn_commit( txi );
 922                         mdb_writes = 0;
 923                         for ( i=0; i<mi->mi_nattrs; i++ )
 924                                 mi->mi_attrs[i]->ai_cursor = NULL;
 925                         if( rc != 0 ) {
 926                                 Debug( LDAP_DEBUG_ANY,
 927                                         "=> " LDAP_XSTRING(mdb_tool_entry_reindex)
 928                                         ": txn_commit failed: %s (%d)\n",
 929                                         mdb_strerror(rc), rc, 0 );
 930                                 e->e_id = NOID;
 931                         }
 932                         mdb_cursor_close( cursor );
 933                         txi = NULL;
 934                         /* Must close the read txn to allow old pages to be reclaimed. */
 935                         mdb_txn_abort( mdb_tool_txn );
 936                         /* and then reopen it so that tool_entry_next still works. */
 937                         mdb_txn_begin( mi->mi_dbenv, NULL, MDB_RDONLY, &mdb_tool_txn );
 938                         mdb_cursor_open( mdb_tool_txn, mi->mi_id2entry, &cursor );
 939                         key.mv_data = &id;
 940                         key.mv_size = sizeof(ID);
 941                         mdb_cursor_get( cursor, &key, NULL, MDB_SET );
 942                 }
 943
 944         } else {
 945                 unsigned i;
 946                 mdb_writes = 0;
 947                 mdb_cursor_close( cursor );
 948                 cursor = NULL;
 949                 mdb_txn_abort( txi );
 950                 for ( i=0; i<mi->mi_nattrs; i++ )
 951                         mi->mi_attrs[i]->ai_cursor = NULL;
 952                 Debug( LDAP_DEBUG_ANY,
 953                         "=> " LDAP_XSTRING(mdb_tool_entry_reindex)
 954                         ": txn_aborted! err=%d\n",
 955                         rc, 0, 0 );
 956                 e->e_id = NOID;
 957                 txi = NULL;
 958         }
 959         mdb_entry_release( &op, e, 0 );
 960
 961         return rc;
 962 }
 963
 964 ID mdb_tool_entry_modify(
 965         BackendDB *be,
 966         Entry *e,
 967         struct berval *text )
 968 {
 969         int rc;
 970         struct mdb_info *mdb;
 971         Operation op = {0};
 972         Opheader ohdr = {0};
 973
 974         assert( be != NULL );
 975         assert( slapMode & SLAP_TOOL_MODE );
 976
 977         assert( text != NULL );
 978         assert( text->bv_val != NULL );
 979         assert( text->bv_val[0] == '\0' );      /* overconservative? */
 980
 981         assert ( e->e_id != NOID );
 982
 983         Debug( LDAP_DEBUG_TRACE,
 984                 "=> " LDAP_XSTRING(mdb_tool_entry_modify) "( %ld, \"%s\" )\n",
 985                 (long) e->e_id, e->e_dn, 0 );
 986
 987         mdb = (struct mdb_info *) be->be_private;
 988
 989         if( cursor ) {
 990                 mdb_cursor_close( cursor );
 991                 cursor = NULL;
 992         }
 993         if ( !mdb_tool_txn ) {
 994                 rc = mdb_txn_begin( mdb->mi_dbenv, NULL, 0, &mdb_tool_txn );
 995                 if( rc != 0 ) {
 996                         snprintf( text->bv_val, text->bv_len,
 997                                 "txn_begin failed: %s (%d)",
 998                                 mdb_strerror(rc), rc );
 999                         Debug( LDAP_DEBUG_ANY,
1000                                 "=> " LDAP_XSTRING(mdb_tool_entry_modify) ": %s\n",
1001                                  text->bv_val, 0, 0 );
1002                         return NOID;
1003                 }
1004         }
1005
1006         op.o_hdr = &ohdr;
1007         op.o_bd = be;
1008         op.o_tmpmemctx = NULL;
1009         op.o_tmpmfuncs = &ch_mfuncs;
1010
1011         /* id2entry index */
1012         rc = mdb_id2entry_update( &op, mdb_tool_txn, NULL, e );
1013         if( rc != 0 ) {
1014                 snprintf( text->bv_val, text->bv_len,
1015                                 "id2entry_update failed: err=%d", rc );
1016                 Debug( LDAP_DEBUG_ANY,
1017                         "=> " LDAP_XSTRING(mdb_tool_entry_modify) ": %s\n",
1018                         text->bv_val, 0, 0 );
1019                 goto done;
1020         }
1021
1022 done:
1023         if( rc == 0 ) {
1024                 rc = mdb_txn_commit( mdb_tool_txn );
1025                 if( rc != 0 ) {
1026                         mdb->mi_numads = 0;
1027                         snprintf( text->bv_val, text->bv_len,
1028                                         "txn_commit failed: %s (%d)",
1029                                         mdb_strerror(rc), rc );
1030                         Debug( LDAP_DEBUG_ANY,
1031                                 "=> " LDAP_XSTRING(mdb_tool_entry_modify) ": "
1032                                 "%s\n", text->bv_val, 0, 0 );
1033                         e->e_id = NOID;
1034                 }
1035
1036         } else {
1037                 mdb_txn_abort( mdb_tool_txn );
1038                 snprintf( text->bv_val, text->bv_len,
1039                         "txn_aborted! %s (%d)",
1040                         mdb_strerror(rc), rc );
1041                 Debug( LDAP_DEBUG_ANY,
1042                         "=> " LDAP_XSTRING(mdb_tool_entry_modify) ": %s\n",
1043                         text->bv_val, 0, 0 );
1044                 e->e_id = NOID;
1045         }
1046         mdb_tool_txn = NULL;
1047
1048         return e->e_id;
1049 }
1050
1051 int mdb_tool_entry_delete(
1052         BackendDB *be,
1053         struct berval *ndn,
1054         struct berval *text )
1055 {
1056         int rc;
1057         struct mdb_info *mdb;
1058         Operation op = {0};
1059         Opheader ohdr = {0};
1060         Entry *e;
1061
1062         assert( be != NULL );
1063         assert( slapMode & SLAP_TOOL_MODE );
1064
1065         assert( text != NULL );
1066         assert( text->bv_val != NULL );
1067         assert( text->bv_val[0] == '\0' );      /* overconservative? */
1068
1069         assert ( ndn != NULL );
1070         assert ( ndn->bv_val != NULL );
1071
1072         Debug( LDAP_DEBUG_TRACE,
1073                 "=> " LDAP_XSTRING(mdb_tool_entry_delete) "( %s )\n",
1074                 ndn->bv_val, 0, 0 );
1075
1076         mdb = (struct mdb_info *) be->be_private;
1077
1078         assert( cursor == NULL );
1079         if( cursor ) {
1080                 mdb_cursor_close( cursor );
1081                 cursor = NULL;
1082         }
1083         if( !mdb_tool_txn ) {
1084                 rc = mdb_txn_begin( mdb->mi_dbenv, NULL, 0, &mdb_tool_txn );
1085                 if( rc != 0 ) {
1086                         snprintf( text->bv_val, text->bv_len,
1087                                 "txn_begin failed: %s (%d)",
1088                                 mdb_strerror(rc), rc );
1089                         Debug( LDAP_DEBUG_ANY,
1090                                 "=> " LDAP_XSTRING(mdb_tool_entry_delete) ": %s\n",
1091                                  text->bv_val, 0, 0 );
1092                         return LDAP_OTHER;
1093                 }
1094         }
1095
1096         rc = mdb_cursor_open( mdb_tool_txn, mdb->mi_dn2id, &cursor );
1097         if( rc != 0 ) {
1098                 snprintf( text->bv_val, text->bv_len,
1099                         "cursor_open failed: %s (%d)",
1100                         mdb_strerror(rc), rc );
1101                 Debug( LDAP_DEBUG_ANY,
1102                         "=> " LDAP_XSTRING(mdb_tool_entry_delete) ": %s\n",
1103                          text->bv_val, 0, 0 );
1104                 return LDAP_OTHER;
1105         }
1106
1107         op.o_hdr = &ohdr;
1108         op.o_bd = be;
1109         op.o_tmpmemctx = NULL;
1110         op.o_tmpmfuncs = &ch_mfuncs;
1111
1112         rc = mdb_dn2entry( &op, mdb_tool_txn, cursor, ndn, &e, NULL, 0 );
1113         if( rc != 0 ) {
1114                 snprintf( text->bv_val, text->bv_len,
1115                         "dn2entry failed: %s (%d)",
1116                         mdb_strerror(rc), rc );
1117                 Debug( LDAP_DEBUG_ANY,
1118                         "=> " LDAP_XSTRING(mdb_tool_entry_delete) ": %s\n",
1119                          text->bv_val, 0, 0 );
1120                 goto done;
1121         }
1122
1123         /* check that we wouldn't orphan any children */
1124         rc = mdb_dn2id_children( &op, mdb_tool_txn, e );
1125         if( rc != MDB_NOTFOUND ) {
1126                 switch( rc ) {
1127                 case 0:
1128                         snprintf( text->bv_val, text->bv_len,
1129                                 "delete failed:"
1130                                 " subordinate objects must be deleted first");
1131                         break;
1132                 default:
1133                         snprintf( text->bv_val, text->bv_len,
1134                                 "has_children failed: %s (%d)",
1135                                 mdb_strerror(rc), rc );
1136                         break;
1137                 }
1138                 rc = -1;
1139                 Debug( LDAP_DEBUG_ANY,
1140                         "=> " LDAP_XSTRING(mdb_tool_entry_delete) ": %s\n",
1141                          text->bv_val, 0, 0 );
1142                 goto done;
1143         }
1144
1145         /* delete from dn2id */
1146         rc = mdb_dn2id_delete( &op, cursor, e->e_id, 1 );
1147         if( rc != 0 ) {
1148                 snprintf( text->bv_val, text->bv_len,
1149                                 "dn2id_delete failed: err=%d", rc );
1150                 Debug( LDAP_DEBUG_ANY,
1151                         "=> " LDAP_XSTRING(mdb_tool_entry_delete) ": %s\n",
1152                         text->bv_val, 0, 0 );
1153                 goto done;
1154         }
1155
1156         /* deindex values */
1157         rc = mdb_index_entry_del( &op, mdb_tool_txn, e );
1158         if( rc != 0 ) {
1159                 snprintf( text->bv_val, text->bv_len,
1160                                 "entry_delete failed: err=%d", rc );
1161                 Debug( LDAP_DEBUG_ANY,
1162                         "=> " LDAP_XSTRING(mdb_tool_entry_delete) ": %s\n",
1163                         text->bv_val, 0, 0 );
1164                 goto done;
1165         }
1166
1167         /* do the deletion */
1168         rc = mdb_id2entry_delete( be, mdb_tool_txn, e );
1169         if( rc != 0 ) {
1170                 snprintf( text->bv_val, text->bv_len,
1171                                 "id2entry_update failed: err=%d", rc );
1172                 Debug( LDAP_DEBUG_ANY,
1173                         "=> " LDAP_XSTRING(mdb_tool_entry_delete) ": %s\n",
1174                         text->bv_val, 0, 0 );
1175                 goto done;
1176         }
1177
1178 done:
1179         /* free entry */
1180         if( e != NULL ) {
1181                 mdb_entry_return( &op, e );
1182         }
1183
1184         if( rc == 0 ) {
1185                 rc = mdb_txn_commit( mdb_tool_txn );
1186                 if( rc != 0 ) {
1187                         snprintf( text->bv_val, text->bv_len,
1188                                         "txn_commit failed: %s (%d)",
1189                                         mdb_strerror(rc), rc );
1190                         Debug( LDAP_DEBUG_ANY,
1191                                 "=> " LDAP_XSTRING(mdb_tool_entry_delete) ": "
1192                                 "%s\n", text->bv_val, 0, 0 );
1193                 }
1194
1195         } else {
1196                 mdb_txn_abort( mdb_tool_txn );
1197                 snprintf( text->bv_val, text->bv_len,
1198                         "txn_aborted! %s (%d)",
1199                         mdb_strerror(rc), rc );
1200                 Debug( LDAP_DEBUG_ANY,
1201                         "=> " LDAP_XSTRING(mdb_tool_entry_delete) ": %s\n",
1202                         text->bv_val, 0, 0 );
1203         }
1204         mdb_tool_txn = NULL;
1205         cursor = NULL;
1206
1207         return rc;
1208 }
1209
1210 static void *
1211 mdb_tool_index_task( void *ctx, void *ptr )
1212 {
1213         int base = *(int *)ptr;
1214         Operation op = {0};
1215         Opheader ohdr = {0};
1216         AttrIxInfo ai = {0}, *aio;
1217
1218         free( ptr );
1219         op.o_hdr = &ohdr;
1220         op.o_bd = mdb_tool_ix_be;
1221         op.o_tmpmemctx = NULL;
1222         op.o_tmpmfuncs = &ch_mfuncs;
1223         aio = mdb_tool_axinfo[base];
1224         mdb_tool_axinfo[base] = &ai;
1225         LDAP_SLIST_INSERT_HEAD( &op.o_extra, &ai.ai_oe, oe_next );
1226         while ( 1 ) {
1227                 ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex );
1228                 mdb_tool_index_tcount--;
1229                 if ( !mdb_tool_index_tcount )
1230                         ldap_pvt_thread_cond_signal( &mdb_tool_index_cond_main );
1231                 ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_work,
1232                         &mdb_tool_index_mutex );
1233                 if ( slapd_shutdown ) {
1234                         mdb_tool_index_tcount--;
1235                         if ( !mdb_tool_index_tcount )
1236                                 ldap_pvt_thread_cond_signal( &mdb_tool_index_cond_main );
1237                         *aio = ai;
1238                         mdb_tool_axinfo[base] = aio;
1239                         ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex );
1240                         break;
1241                 }
1242                 ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex );
1243                 mdb_tool_index_rec[base].ir_i = mdb_index_recrun( &op,
1244                         mdb_tool_ix_txn,
1245                         mdb_tool_info, mdb_tool_index_rec, mdb_tool_ix_id, base );
1246         }
1247
1248         return NULL;
1249 }
1250
1251 #ifdef MDB_TOOL_IDL_CACHING
1252 static int
1253 mdb_tool_idl_cmp( const void *v1, const void *v2 )
1254 {
1255         const mdb_tool_idl_cache *c1 = v1, *c2 = v2;
1256         int rc;
1257
1258         if (( rc = c1->kstr.bv_len - c2->kstr.bv_len )) return rc;
1259         return memcmp( c1->kstr.bv_val, c2->kstr.bv_val, c1->kstr.bv_len );
1260 }
1261
1262 static int
1263 mdb_tool_idl_flush_one( MDB_cursor *mc, AttrIxInfo *ai, mdb_tool_idl_cache *ic )
1264 {
1265         mdb_tool_idl_cache_entry *ice;
1266         MDB_val key, data[2];
1267         int i, rc;
1268         ID id, nid;
1269
1270         /* Freshly allocated, ignore it */
1271         if ( !ic->head && ic->count <= MDB_IDL_DB_SIZE ) {
1272                 return 0;
1273         }
1274
1275         key.mv_data = ic->kstr.bv_val;
1276         key.mv_size = ic->kstr.bv_len;
1277
1278         if ( ic->count > MDB_IDL_DB_SIZE ) {
1279                 while ( ic->flags & WAS_FOUND ) {
1280                         rc = mdb_cursor_get( mc, &key, data, MDB_SET );
1281                         if ( rc ) {
1282                                 /* FIXME: find out why this happens */
1283                                 ic->flags = 0;
1284                                 break;
1285                         }
1286                         if ( ic->flags & WAS_RANGE ) {
1287                                 /* Skip lo */
1288                                 rc = mdb_cursor_get( mc, &key, data, MDB_NEXT_DUP );
1289
1290                                 /* Get hi */
1291                                 rc = mdb_cursor_get( mc, &key, data, MDB_NEXT_DUP );
1292
1293                                 /* Store range hi */
1294                                 data[0].mv_data = &ic->last;
1295                                 rc = mdb_cursor_put( mc, &key, data, MDB_CURRENT );
1296                         } else {
1297                                 /* Delete old data, replace with range */
1298                                 ic->first = *(ID *)data[0].mv_data;
1299                                 mdb_cursor_del( mc, MDB_NODUPDATA );
1300                         }
1301                         break;
1302                 }
1303                 if ( !(ic->flags & WAS_RANGE)) {
1304                         /* range, didn't exist before */
1305                         nid = 0;
1306                         data[0].mv_size = sizeof(ID);
1307                         data[0].mv_data = &nid;
1308                         rc = mdb_cursor_put( mc, &key, data, 0 );
1309                         if ( rc == 0 ) {
1310                                 data[0].mv_data = &ic->first;
1311                                 rc = mdb_cursor_put( mc, &key, data, 0 );
1312                                 if ( rc == 0 ) {
1313                                         data[0].mv_data = &ic->last;
1314                                         rc = mdb_cursor_put( mc, &key, data, 0 );
1315                                 }
1316                         }
1317                         if ( rc ) {
1318                                 rc = -1;
1319                         }
1320                 }
1321         } else {
1322                 /* Normal write */
1323                 int n;
1324
1325                 data[0].mv_size = sizeof(ID);
1326                 rc = 0;
1327                 for ( ice = ic->head, n=0; ice; ice = ice->next, n++ ) {
1328                         int end;
1329                         if ( ice->next ) {
1330                                 end = IDBLOCK;
1331                         } else {
1332                                 end = (ic->count-ic->offset) & (IDBLOCK-1);
1333                                 if ( !end )
1334                                         end = IDBLOCK;
1335                         }
1336                         data[1].mv_size = end;
1337                         data[0].mv_data = ice->ids;
1338                         rc = mdb_cursor_put( mc, &key, data, MDB_APPENDDUP|MDB_MULTIPLE );
1339                         if ( rc ) {
1340                                 rc = -1;
1341                                 break;
1342                         }
1343                 }
1344                 if ( ic->head ) {
1345                         ic->tail->next = ai->ai_flist;
1346                         ai->ai_flist = ic->head;
1347                 }
1348         }
1349         ic->head = ai->ai_clist;
1350         ai->ai_clist = ic;
1351         return rc;
1352 }
1353
1354 static int
1355 mdb_tool_idl_flush_db( MDB_txn *txn, AttrInfo *ai, AttrIxInfo *ax )
1356 {
1357         MDB_cursor *mc;
1358         Avlnode *root;
1359         int rc;
1360
1361         mdb_cursor_open( txn, ai->ai_dbi, &mc );
1362         root = tavl_end( ai->ai_root, TAVL_DIR_LEFT );
1363         do {
1364                 rc = mdb_tool_idl_flush_one( mc, ax, root->avl_data );
1365                 if ( rc != -1 )
1366                         rc = 0;
1367         } while ((root = tavl_next(root, TAVL_DIR_RIGHT)));
1368         mdb_cursor_close( mc );
1369
1370         return rc;
1371 }
1372
1373 static int
1374 mdb_tool_idl_flush( BackendDB *be, MDB_txn *txn )
1375 {
1376         struct mdb_info *mdb = (struct mdb_info *) be->be_private;
1377         int rc = 0;
1378         unsigned int i, dbi;
1379
1380         for ( i=0; i < mdb->mi_nattrs; i++ ) {
1381                 if ( !mdb->mi_attrs[i]->ai_root ) continue;
1382                 rc = mdb_tool_idl_flush_db( txn, mdb->mi_attrs[i], mdb_tool_axinfo[i % mdb_tool_threads] );
1383                 tavl_free(mdb->mi_attrs[i]->ai_root, NULL);
1384                 mdb->mi_attrs[i]->ai_root = NULL;
1385                 if ( rc )
1386                         break;
1387         }
1388         return rc;
1389 }
1390
1391 int mdb_tool_idl_add(
1392         BackendDB *be,
1393         MDB_cursor *mc,
1394         struct berval *keys,
1395         ID id )
1396 {
1397         MDB_dbi dbi;
1398         mdb_tool_idl_cache *ic, itmp;
1399         mdb_tool_idl_cache_entry *ice;
1400         int i, rc, lcount;
1401         AttrIxInfo *ax = (AttrIxInfo *)mc;
1402         AttrInfo *ai = (AttrInfo *)ax->ai_ai;
1403         mc = ai->ai_cursor;
1404
1405         dbi = ai->ai_dbi;
1406         for (i=0; keys[i].bv_val; i++) {
1407         itmp.kstr = keys[i];
1408         ic = tavl_find( ai->ai_root, &itmp, mdb_tool_idl_cmp );
1409
1410         /* No entry yet, create one */
1411         if ( !ic ) {
1412                 MDB_val key, data;
1413                 ID nid;
1414                 int rc;
1415
1416                 if ( ax->ai_clist ) {
1417                         ic = ax->ai_clist;
1418                         ax->ai_clist = ic->head;
1419                 } else {
1420                         ic = ch_malloc( sizeof( mdb_tool_idl_cache ) + itmp.kstr.bv_len + 4 );
1421                 }
1422                 ic->kstr.bv_len = itmp.kstr.bv_len;
1423                 ic->kstr.bv_val = (char *)(ic+1);
1424                 memcpy( ic->kstr.bv_val, itmp.kstr.bv_val, ic->kstr.bv_len );
1425                 ic->head = ic->tail = NULL;
1426                 ic->last = 0;
1427                 ic->count = 0;
1428                 ic->offset = 0;
1429                 ic->flags = 0;
1430                 tavl_insert( &ai->ai_root, ic, mdb_tool_idl_cmp,
1431                         avl_dup_error );
1432
1433                 /* load existing key count here */
1434                 key.mv_size = keys[i].bv_len;
1435                 key.mv_data = keys[i].bv_val;
1436                 rc = mdb_cursor_get( mc, &key, &data, MDB_SET );
1437                 if ( rc == 0 ) {
1438                         ic->flags |= WAS_FOUND;
1439                         nid = *(ID *)data.mv_data;
1440                         if ( nid == 0 ) {
1441                                 ic->count = MDB_IDL_DB_SIZE+1;
1442                                 ic->flags |= WAS_RANGE;
1443                         } else {
1444                                 size_t count;
1445
1446                                 mdb_cursor_count( mc, &count );
1447                                 ic->count = count;
1448                                 ic->first = nid;
1449                                 ic->offset = count & (IDBLOCK-1);
1450                         }
1451                 }
1452         }
1453         /* are we a range already? */
1454         if ( ic->count > MDB_IDL_DB_SIZE ) {
1455                 ic->last = id;
1456                 continue;
1457         /* Are we at the limit, and converting to a range? */
1458         } else if ( ic->count == MDB_IDL_DB_SIZE ) {
1459                 if ( ic->head ) {
1460                         ic->tail->next = ax->ai_flist;
1461                         ax->ai_flist = ic->head;
1462                 }
1463                 ic->head = ic->tail = NULL;
1464                 ic->last = id;
1465                 ic->count++;
1466                 continue;
1467         }
1468         /* No free block, create that too */
1469         lcount = (ic->count-ic->offset) & (IDBLOCK-1);
1470         if ( !ic->tail || lcount == 0) {
1471                 if ( ax->ai_flist ) {
1472                         ice = ax->ai_flist;
1473                         ax->ai_flist = ice->next;
1474                 } else {
1475                         ice = ch_malloc( sizeof( mdb_tool_idl_cache_entry ));
1476                 }
1477                 ice->next = NULL;
1478                 if ( !ic->head ) {
1479                         ic->head = ice;
1480                 } else {
1481                         ic->tail->next = ice;
1482                 }
1483                 ic->tail = ice;
1484                 if ( lcount )
1485                         ice->ids[lcount-1] = 0;
1486                 if ( !ic->count )
1487                         ic->first = id;
1488         }
1489         ice = ic->tail;
1490         if (!lcount || ice->ids[lcount-1] != id) {
1491                 ice->ids[lcount] = id;
1492                 ic->count++;
1493         }
1494         }
1495
1496         return 0;
1497 }
1498 #endif /* MDB_TOOL_IDL_CACHING */
1499
1500 /* Upgrade from pre 2.4.34 dn2id format */
1501
1502 #include <ac/unistd.h>
1503 #include <lutil_meter.h>
1504
1505 #define STACKSIZ        2048
1506
1507 typedef struct rec {
1508         ID id;
1509         size_t len;
1510         char rdn[512];
1511 } rec;
1512
1513 static int
1514 mdb_dn2id_upgrade( BackendDB *be ) {
1515         struct mdb_info *mi = (struct mdb_info *) be->be_private;
1516         MDB_txn *mt;
1517         MDB_cursor *mc = NULL;
1518         MDB_val key, data;
1519         int rc, writes=0, depth=0;
1520         int enable_meter = 0;
1521         ID id = 0, *num, count = 0;
1522         rec *stack;
1523         lutil_meter_t meter;
1524
1525         if (!(mi->mi_flags & MDB_NEED_UPGRADE)) {
1526                 Debug( LDAP_DEBUG_ANY, "database %s: No upgrade needed.\n",
1527                         be->be_suffix[0].bv_val, 0, 0 );
1528                 return 0;
1529         }
1530
1531         {
1532                 MDB_stat st;
1533
1534                 mdb_stat(mdb_cursor_txn(cursor), mi->mi_dbis[MDB_ID2ENTRY], &st);
1535                 if (!st.ms_entries) {
1536                         /* Empty DB, nothing to upgrade? */
1537                         return 0;
1538                 }
1539                 if (isatty(2))
1540                         enable_meter = !lutil_meter_open(&meter,
1541                                 &lutil_meter_text_display,
1542                                 &lutil_meter_linear_estimator,
1543                                 st.ms_entries);
1544         }
1545
1546         num = ch_malloc(STACKSIZ * (sizeof(ID) + sizeof(rec)));
1547         stack = (rec *)(num + STACKSIZ);
1548
1549         rc = mdb_txn_begin(mi->mi_dbenv, NULL, 0, &mt);
1550         if (rc) {
1551                 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_begin failed, %s (%d)\n",
1552                         mdb_strerror(rc), rc, 0 );
1553                 goto leave;
1554         }
1555         rc = mdb_cursor_open(mt, mi->mi_dbis[MDB_DN2ID], &mc);
1556         if (rc) {
1557                 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_open failed, %s (%d)\n",
1558                         mdb_strerror(rc), rc, 0 );
1559                 goto leave;
1560         }
1561
1562         key.mv_size = sizeof(ID);
1563         /* post-order depth-first update */
1564         for(;;) {
1565                 size_t dkids;
1566                 unsigned char *ptr;
1567
1568                 /* visit */
1569                 key.mv_data = &id;
1570                 stack[depth].id = id;
1571                 rc = mdb_cursor_get(mc, &key, &data, MDB_SET);
1572                 if (rc) {
1573                         Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_get failed, %s (%d)\n",
1574                                 mdb_strerror(rc), rc, 0 );
1575                         goto leave;
1576                 }
1577                 num[depth] = 1;
1578
1579                 rc = mdb_cursor_count(mc, &dkids);
1580                 if (rc) {
1581                         Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_count failed, %s (%d)\n",
1582                                 mdb_strerror(rc), rc, 0 );
1583                         goto leave;
1584                 }
1585                 if (dkids > 1) {
1586                         rc = mdb_cursor_get(mc, &key, &data, MDB_NEXT_DUP);
1587 down:
1588                         ptr = (unsigned char *)data.mv_data + data.mv_size - sizeof(ID);
1589                         memcpy(&id, ptr, sizeof(ID));
1590                         depth++;
1591                         memcpy(stack[depth].rdn, data.mv_data, data.mv_size);
1592                         stack[depth].len = data.mv_size;
1593                         continue;
1594                 }
1595
1596
1597                 /* pop: write updated count, advance to next node */
1598 pop:
1599                 /* update superior counts */
1600                 if (depth)
1601                         num[depth-1] += num[depth];
1602
1603                 key.mv_data = &id;
1604                 id = stack[depth-1].id;
1605                 data.mv_data = stack[depth].rdn;
1606                 data.mv_size = stack[depth].len;
1607                 rc = mdb_cursor_get(mc, &key, &data, MDB_GET_BOTH);
1608                 if (rc) {
1609                         Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_get(BOTH) failed, %s (%d)\n",
1610                                 mdb_strerror(rc), rc, 0 );
1611                         goto leave;
1612                 }
1613                 data.mv_data = stack[depth].rdn;
1614                 ptr = (unsigned char *)data.mv_data + data.mv_size;
1615                 memcpy(ptr, &num[depth], sizeof(ID));
1616                 data.mv_size += sizeof(ID);
1617                 rc = mdb_cursor_del(mc, 0);
1618                 if (rc) {
1619                         Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_del failed, %s (%d)\n",
1620                                 mdb_strerror(rc), rc, 0 );
1621                         goto leave;
1622                 }
1623                 rc = mdb_cursor_put(mc, &key, &data, 0);
1624                 if (rc) {
1625                         Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_put failed, %s (%d)\n",
1626                                 mdb_strerror(rc), rc, 0 );
1627                         goto leave;
1628                 }
1629                 count++;
1630 #if 1
1631                 if (enable_meter)
1632                         lutil_meter_update(&meter, count, 0);
1633 #else
1634                 {
1635                         int len;
1636                         ptr = data.mv_data;
1637                         len = (ptr[0] & 0x7f) << 8 | ptr[1];
1638                         printf("ID: %zu, %zu, %.*s\n", stack[depth].id, num[depth], len, ptr+2);
1639                 }
1640 #endif
1641                 writes++;
1642                 if (writes == 1000) {
1643                         mdb_cursor_close(mc);
1644                         rc = mdb_txn_commit(mt);
1645                         if (rc) {
1646                                 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_commit failed, %s (%d)\n",
1647                                         mdb_strerror(rc), rc, 0 );
1648                                 goto leave;
1649                         }
1650                         rc = mdb_txn_begin(mi->mi_dbenv, NULL, 0, &mt);
1651                         if (rc) {
1652                                 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_begin(2) failed, %s (%d)\n",
1653                                         mdb_strerror(rc), rc, 0 );
1654                                 goto leave;
1655                         }
1656                         rc = mdb_cursor_open(mt, mi->mi_dbis[MDB_DN2ID], &mc);
1657                         if (rc) {
1658                                 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_open(2) failed, %s (%d)\n",
1659                                         mdb_strerror(rc), rc, 0 );
1660                                 goto leave;
1661                         }
1662                         rc = mdb_cursor_get(mc, &key, &data, MDB_GET_BOTH);
1663                         if (rc) {
1664                                 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_get(2) failed, %s (%d)\n",
1665                                         mdb_strerror(rc), rc, 0 );
1666                                 goto leave;
1667                         }
1668                         writes = 0;
1669                 }
1670                 depth--;
1671
1672                 rc = mdb_cursor_get(mc, &key, &data, MDB_NEXT_DUP);
1673                 if (rc == 0)
1674                         goto down;
1675                 rc = 0;
1676                 if (depth)
1677                         goto pop;
1678                 else
1679                         break;
1680         }
1681 leave:
1682         mdb_cursor_close(mc);
1683         if (mt) {
1684                 int r2;
1685                 r2 = mdb_txn_commit(mt);
1686                 if (r2) {
1687                         Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_commit(2) failed, %s (%d)\n",
1688                                 mdb_strerror(r2), r2, 0 );
1689                         if (!rc)
1690                                 rc = r2;
1691                 }
1692         }
1693         ch_free(num);
1694         if (enable_meter) {
1695                 lutil_meter_update(&meter, count, 1);
1696                 lutil_meter_close(&meter);
1697         }
1698         return rc;
1699 }