git.sur5r.net Git - openldap/blob - servers/slapd/back-mdb/tools.c

   1 /* tools.c - tools for slap tools */
   2 /* $OpenLDAP$ */
   3 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
   4  *
   5  * Copyright 2011-2017 The OpenLDAP Foundation.
   6  * All rights reserved.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted only as authorized by the OpenLDAP
  10  * Public License.
  11  *
  12  * A copy of this license is available in the file LICENSE in the
  13  * top-level directory of the distribution or, alternatively, at
  14  * <http://www.OpenLDAP.org/license.html>.
  15  */
  16
  17 #include "portable.h"
  18
  19 #include <stdio.h>
  20 #include <ac/string.h>
  21 #include <ac/errno.h>
  22
  23 #define AVL_INTERNAL
  24 #include "back-mdb.h"
  25 #include "idl.h"
  26
  27 #ifdef MDB_TOOL_IDL_CACHING
  28 static int mdb_tool_idl_flush( BackendDB *be, MDB_txn *txn );
  29
  30 #define IDBLOCK 1024
  31
  32 typedef struct mdb_tool_idl_cache_entry {
  33         struct mdb_tool_idl_cache_entry *next;
  34         ID ids[IDBLOCK];
  35 } mdb_tool_idl_cache_entry;
  36
  37 typedef struct mdb_tool_idl_cache {
  38         struct berval kstr;
  39         mdb_tool_idl_cache_entry *head, *tail;
  40         ID first, last;
  41         int count;
  42         short offset;
  43         short flags;
  44 } mdb_tool_idl_cache;
  45 #define WAS_FOUND       0x01
  46 #define WAS_RANGE       0x02
  47
  48 #define MDB_TOOL_IDL_FLUSH(be, txn)     mdb_tool_idl_flush(be, txn)
  49 #else
  50 #define MDB_TOOL_IDL_FLUSH(be, txn)
  51 #endif /* MDB_TOOL_IDL_CACHING */
  52
  53 MDB_txn *mdb_tool_txn = NULL;
  54
  55 static MDB_txn *txi = NULL;
  56 static MDB_cursor *cursor = NULL, *idcursor = NULL;
  57 static MDB_cursor *mcp = NULL, *mcd = NULL;
  58 static MDB_val key, data;
  59 static ID previd = NOID;
  60
  61 typedef struct dn_id {
  62         ID id;
  63         struct berval dn;
  64 } dn_id;
  65
  66 #define HOLE_SIZE       4096
  67 static dn_id hbuf[HOLE_SIZE], *holes = hbuf;
  68 static unsigned nhmax = HOLE_SIZE;
  69 static unsigned nholes;
  70
  71 static struct berval    *tool_base;
  72 static int              tool_scope;
  73 static Filter           *tool_filter;
  74 static Entry            *tool_next_entry;
  75
  76 static ID mdb_tool_ix_id;
  77 static BackendDB *mdb_tool_ix_be;
  78 static MDB_txn *mdb_tool_ix_txn;
  79 static int mdb_tool_index_tcount, mdb_tool_threads;
  80 static IndexRec *mdb_tool_index_rec;
  81 static AttrIxInfo **mdb_tool_axinfo;
  82 static struct mdb_info *mdb_tool_info;
  83 static ldap_pvt_thread_mutex_t mdb_tool_index_mutex;
  84 static ldap_pvt_thread_cond_t mdb_tool_index_cond_main;
  85 static ldap_pvt_thread_cond_t mdb_tool_index_cond_work;
  86 static void * mdb_tool_index_task( void *ctx, void *ptr );
  87
  88 static int      mdb_writes, mdb_writes_per_commit;
  89
  90 /* Number of ops per commit in Quick mode.
  91  * Batching speeds writes overall, but too large a
  92  * batch will fail with MDB_TXN_FULL.
  93  */
  94 #ifndef MDB_WRITES_PER_COMMIT
  95 #define MDB_WRITES_PER_COMMIT   500
  96 #endif
  97
  98 static int
  99 mdb_tool_entry_get_int( BackendDB *be, ID id, Entry **ep );
 100
 101 int mdb_tool_entry_open(
 102         BackendDB *be, int mode )
 103 {
 104         /* In Quick mode, commit once per 500 entries */
 105         mdb_writes = 0;
 106         if ( slapMode & SLAP_TOOL_QUICK )
 107                 mdb_writes_per_commit = MDB_WRITES_PER_COMMIT;
 108         else
 109                 mdb_writes_per_commit = 1;
 110
 111 #ifdef MDB_TOOL_IDL_CACHING                     /* threaded indexing has no performance advantage */
 112         /* Set up for threaded slapindex */
 113         if (( slapMode & (SLAP_TOOL_QUICK|SLAP_TOOL_READONLY)) == SLAP_TOOL_QUICK ) {
 114                 if ( !mdb_tool_info ) {
 115                         struct mdb_info *mdb = (struct mdb_info *) be->be_private;
 116                         ldap_pvt_thread_mutex_init( &mdb_tool_index_mutex );
 117                         ldap_pvt_thread_cond_init( &mdb_tool_index_cond_main );
 118                         ldap_pvt_thread_cond_init( &mdb_tool_index_cond_work );
 119                         if ( mdb->mi_nattrs ) {
 120                                 int i;
 121                                 mdb_tool_threads = slap_tool_thread_max - 1;
 122                                 if ( mdb_tool_threads > 1 ) {
 123                                         mdb_tool_index_rec = ch_calloc( mdb->mi_nattrs, sizeof( IndexRec ));
 124                                         mdb_tool_axinfo = ch_calloc( mdb_tool_threads, sizeof( AttrIxInfo* ) +
 125                                                 sizeof( AttrIxInfo ));
 126                                         mdb_tool_axinfo[0] = (AttrIxInfo *)(mdb_tool_axinfo + mdb_tool_threads);
 127                                         for (i=1; i<mdb_tool_threads; i++)
 128                                                 mdb_tool_axinfo[i] = mdb_tool_axinfo[i-1]+1;
 129                                         mdb_tool_index_tcount = mdb_tool_threads - 1;
 130                                         mdb_tool_ix_be = be;
 131                                         for (i=1; i<mdb_tool_threads; i++) {
 132                                                 int *ptr = ch_malloc( sizeof( int ));
 133                                                 *ptr = i;
 134                                                 ldap_pvt_thread_pool_submit( &connection_pool,
 135                                                         mdb_tool_index_task, ptr );
 136                                         }
 137                                         mdb_tool_info = mdb;
 138                                 }
 139                         }
 140                 }
 141         }
 142 #endif
 143
 144         return 0;
 145 }
 146
 147 int mdb_tool_entry_close(
 148         BackendDB *be )
 149 {
 150 #ifdef MDB_TOOL_IDL_CACHING
 151         if ( mdb_tool_info ) {
 152                 int i;
 153                 slapd_shutdown = 1;
 154                 ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex );
 155
 156                 /* There might still be some threads starting */
 157                 while ( mdb_tool_index_tcount > 0 ) {
 158                         ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main,
 159                                         &mdb_tool_index_mutex );
 160                 }
 161
 162                 mdb_tool_index_tcount = mdb_tool_threads - 1;
 163                 ldap_pvt_thread_cond_broadcast( &mdb_tool_index_cond_work );
 164
 165                 /* Make sure all threads are stopped */
 166                 while ( mdb_tool_index_tcount > 0 ) {
 167                         ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main,
 168                                 &mdb_tool_index_mutex );
 169                 }
 170                 ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex );
 171
 172                 mdb_tool_info = NULL;
 173                 slapd_shutdown = 0;
 174                 ch_free( mdb_tool_index_rec );
 175                 mdb_tool_index_tcount = mdb_tool_threads - 1;
 176                 if (mdb_tool_txn)
 177                         MDB_TOOL_IDL_FLUSH( be, mdb_tool_txn );
 178                 for (i=0; i<mdb_tool_threads; i++) {
 179                         mdb_tool_idl_cache *ic;
 180                         mdb_tool_idl_cache_entry *ice;
 181                         while ((ic = mdb_tool_axinfo[i]->ai_clist)) {
 182                                 mdb_tool_axinfo[i]->ai_clist = ic->head;
 183                                 free(ic);
 184                         }
 185                         while ((ice = mdb_tool_axinfo[i]->ai_flist)) {
 186                                 mdb_tool_axinfo[i]->ai_flist = ice->next;
 187                                 free(ice);
 188                         }
 189                 }
 190         }
 191 #endif
 192
 193         if( idcursor ) {
 194                 mdb_cursor_close( idcursor );
 195                 idcursor = NULL;
 196         }
 197         if( cursor ) {
 198                 mdb_cursor_close( cursor );
 199                 cursor = NULL;
 200         }
 201         {
 202                 struct mdb_info *mdb = be->be_private;
 203                 if ( mdb ) {
 204                         int i;
 205                         for (i=0; i<mdb->mi_nattrs; i++)
 206                                 mdb->mi_attrs[i]->ai_cursor = NULL;
 207                 }
 208         }
 209         if( mdb_tool_txn ) {
 210                 int rc;
 211                 if (( rc = mdb_txn_commit( mdb_tool_txn ))) {
 212                         Debug( LDAP_DEBUG_ANY,
 213                                 LDAP_XSTRING(mdb_tool_entry_close) ": database %s: "
 214                                 "txn_commit failed: %s (%d)\n",
 215                                 be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
 216                         return -1;
 217                 }
 218                 mdb_tool_txn = NULL;
 219         }
 220
 221         if( nholes ) {
 222                 unsigned i;
 223                 fprintf( stderr, "Error, entries missing!\n");
 224                 for (i=0; i<nholes; i++) {
 225                         fprintf(stderr, "  entry %ld: %s\n",
 226                                 holes[i].id, holes[i].dn.bv_val);
 227                 }
 228                 nholes = 0;
 229                 return -1;
 230         }
 231
 232         return 0;
 233 }
 234
 235 ID
 236 mdb_tool_entry_first_x(
 237         BackendDB *be,
 238         struct berval *base,
 239         int scope,
 240         Filter *f )
 241 {
 242         tool_base = base;
 243         tool_scope = scope;
 244         tool_filter = f;
 245
 246         return mdb_tool_entry_next( be );
 247 }
 248
 249 ID mdb_tool_entry_next(
 250         BackendDB *be )
 251 {
 252         int rc;
 253         ID id;
 254         struct mdb_info *mdb;
 255
 256         assert( be != NULL );
 257         assert( slapMode & SLAP_TOOL_MODE );
 258
 259         mdb = (struct mdb_info *) be->be_private;
 260         assert( mdb != NULL );
 261
 262         if ( !mdb_tool_txn ) {
 263                 rc = mdb_txn_begin( mdb->mi_dbenv, NULL, MDB_RDONLY, &mdb_tool_txn );
 264                 if ( rc )
 265                         return NOID;
 266                 rc = mdb_cursor_open( mdb_tool_txn, mdb->mi_id2entry, &cursor );
 267                 if ( rc ) {
 268                         mdb_txn_abort( mdb_tool_txn );
 269                         return NOID;
 270                 }
 271         }
 272
 273 next:;
 274         rc = mdb_cursor_get( cursor, &key, &data, MDB_NEXT );
 275
 276         if( rc ) {
 277                 return NOID;
 278         }
 279
 280         previd = *(ID *)key.mv_data;
 281         id = previd;
 282
 283         if ( !data.mv_size )
 284                 goto next;
 285
 286         if ( tool_filter || tool_base ) {
 287                 static Operation op = {0};
 288                 static Opheader ohdr = {0};
 289
 290                 op.o_hdr = &ohdr;
 291                 op.o_bd = be;
 292                 op.o_tmpmemctx = NULL;
 293                 op.o_tmpmfuncs = &ch_mfuncs;
 294
 295                 if ( tool_next_entry ) {
 296                         mdb_entry_release( &op, tool_next_entry, 0 );
 297                         tool_next_entry = NULL;
 298                 }
 299
 300                 rc = mdb_tool_entry_get_int( be, id, &tool_next_entry );
 301                 if ( rc == LDAP_NO_SUCH_OBJECT ) {
 302                         goto next;
 303                 }
 304
 305                 assert( tool_next_entry != NULL );
 306
 307                 if ( tool_filter && test_filter( NULL, tool_next_entry, tool_filter ) != LDAP_COMPARE_TRUE )
 308                 {
 309                         mdb_entry_release( &op, tool_next_entry, 0 );
 310                         tool_next_entry = NULL;
 311                         goto next;
 312                 }
 313         }
 314
 315         return id;
 316 }
 317
 318 ID mdb_tool_dn2id_get(
 319         Backend *be,
 320         struct berval *dn
 321 )
 322 {
 323         struct mdb_info *mdb;
 324         Operation op = {0};
 325         Opheader ohdr = {0};
 326         ID id;
 327         int rc;
 328
 329         if ( BER_BVISEMPTY(dn) )
 330                 return 0;
 331
 332         mdb = (struct mdb_info *) be->be_private;
 333
 334         if ( !mdb_tool_txn ) {
 335                 rc = mdb_txn_begin( mdb->mi_dbenv, NULL, (slapMode & SLAP_TOOL_READONLY) != 0 ?
 336                         MDB_RDONLY : 0, &mdb_tool_txn );
 337                 if ( rc )
 338                         return NOID;
 339         }
 340
 341         op.o_hdr = &ohdr;
 342         op.o_bd = be;
 343         op.o_tmpmemctx = NULL;
 344         op.o_tmpmfuncs = &ch_mfuncs;
 345
 346         rc = mdb_dn2id( &op, mdb_tool_txn, NULL, dn, &id, NULL, NULL, NULL );
 347         if ( rc == MDB_NOTFOUND )
 348                 return NOID;
 349
 350         return id;
 351 }
 352
 353 static int
 354 mdb_tool_entry_get_int( BackendDB *be, ID id, Entry **ep )
 355 {
 356         Operation op = {0};
 357         Opheader ohdr = {0};
 358
 359         Entry *e = NULL;
 360         struct berval dn = BER_BVNULL, ndn = BER_BVNULL;
 361         int rc;
 362
 363         assert( be != NULL );
 364         assert( slapMode & SLAP_TOOL_MODE );
 365
 366         if ( ( tool_filter || tool_base ) && id == previd && tool_next_entry != NULL ) {
 367                 *ep = tool_next_entry;
 368                 tool_next_entry = NULL;
 369                 return LDAP_SUCCESS;
 370         }
 371
 372         if ( id != previd ) {
 373                 key.mv_size = sizeof(ID);
 374                 key.mv_data = &id;
 375                 rc = mdb_cursor_get( cursor, &key, &data, MDB_SET );
 376                 if ( rc ) {
 377                         rc = LDAP_OTHER;
 378                         goto done;
 379                 }
 380         }
 381         if ( !data.mv_size ) {
 382                 rc = LDAP_NO_SUCH_OBJECT;
 383                 goto done;
 384         }
 385
 386         op.o_hdr = &ohdr;
 387         op.o_bd = be;
 388         op.o_tmpmemctx = NULL;
 389         op.o_tmpmfuncs = &ch_mfuncs;
 390         if ( slapMode & SLAP_TOOL_READONLY ) {
 391                 rc = mdb_id2name( &op, mdb_tool_txn, &idcursor, id, &dn, &ndn );
 392                 if ( rc  ) {
 393                         rc = LDAP_OTHER;
 394                         goto done;
 395                 }
 396                 if ( tool_base != NULL ) {
 397                         if ( !dnIsSuffixScope( &ndn, tool_base, tool_scope ) ) {
 398                                 ch_free( dn.bv_val );
 399                                 ch_free( ndn.bv_val );
 400                                 rc = LDAP_NO_SUCH_OBJECT;
 401                                 goto done;
 402                         }
 403                 }
 404         }
 405         rc = mdb_entry_decode( &op, mdb_tool_txn, &data, id, &e );
 406         e->e_id = id;
 407         if ( !BER_BVISNULL( &dn )) {
 408                 e->e_name = dn;
 409                 e->e_nname = ndn;
 410         } else {
 411                 e->e_name.bv_val = NULL;
 412                 e->e_nname.bv_val = NULL;
 413         }
 414
 415 done:
 416         if ( e != NULL ) {
 417                 *ep = e;
 418         }
 419
 420         return rc;
 421 }
 422
 423 Entry*
 424 mdb_tool_entry_get( BackendDB *be, ID id )
 425 {
 426         Entry *e = NULL;
 427         int rc;
 428
 429         if ( !mdb_tool_txn ) {
 430                 struct mdb_info *mdb = (struct mdb_info *) be->be_private;
 431                 rc = mdb_txn_begin( mdb->mi_dbenv, NULL,
 432                         (slapMode & SLAP_TOOL_READONLY) ? MDB_RDONLY : 0, &mdb_tool_txn );
 433                 if ( rc )
 434                         return NULL;
 435         }
 436         if ( !cursor ) {
 437                 struct mdb_info *mdb = (struct mdb_info *) be->be_private;
 438                 rc = mdb_cursor_open( mdb_tool_txn, mdb->mi_id2entry, &cursor );
 439                 if ( rc ) {
 440                         mdb_txn_abort( mdb_tool_txn );
 441                         mdb_tool_txn = NULL;
 442                         return NULL;
 443                 }
 444         }
 445         (void)mdb_tool_entry_get_int( be, id, &e );
 446         return e;
 447 }
 448
 449 static int mdb_tool_next_id(
 450         Operation *op,
 451         MDB_txn *tid,
 452         Entry *e,
 453         struct berval *text,
 454         int hole )
 455 {
 456         struct mdb_info *mdb = (struct mdb_info *) op->o_bd->be_private;
 457         struct berval dn = e->e_name;
 458         struct berval ndn = e->e_nname;
 459         struct berval pdn, npdn, nmatched;
 460         ID id, pid = 0;
 461         int rc;
 462
 463         if (ndn.bv_len == 0) {
 464                 e->e_id = 0;
 465                 return 0;
 466         }
 467
 468         rc = mdb_dn2id( op, tid, mcp, &ndn, &id, NULL, NULL, &nmatched );
 469         if ( rc == MDB_NOTFOUND ) {
 470                 if ( !be_issuffix( op->o_bd, &ndn ) ) {
 471                         ID eid = e->e_id;
 472                         dnParent( &ndn, &npdn );
 473                         if ( nmatched.bv_len != npdn.bv_len ) {
 474                                 dnParent( &dn, &pdn );
 475                                 e->e_name = pdn;
 476                                 e->e_nname = npdn;
 477                                 rc = mdb_tool_next_id( op, tid, e, text, 1 );
 478                                 e->e_name = dn;
 479                                 e->e_nname = ndn;
 480                                 if ( rc ) {
 481                                         return rc;
 482                                 }
 483                                 /* If parent didn't exist, it was created just now
 484                                  * and its ID is now in e->e_id. Make sure the current
 485                                  * entry gets added under the new parent ID.
 486                                  */
 487                                 if ( eid != e->e_id ) {
 488                                         pid = e->e_id;
 489                                 }
 490                         } else {
 491                                 pid = id;
 492                         }
 493                 }
 494                 rc = mdb_next_id( op->o_bd, idcursor, &e->e_id );
 495                 if ( rc ) {
 496                         snprintf( text->bv_val, text->bv_len,
 497                                 "next_id failed: %s (%d)",
 498                                 mdb_strerror(rc), rc );
 499                 Debug( LDAP_DEBUG_ANY,
 500                         "=> mdb_tool_next_id: %s\n", text->bv_val, 0, 0 );
 501                         return rc;
 502                 }
 503                 rc = mdb_dn2id_add( op, mcp, mcd, pid, 1, 1, e );
 504                 if ( rc ) {
 505                         snprintf( text->bv_val, text->bv_len,
 506                                 "dn2id_add failed: %s (%d)",
 507                                 mdb_strerror(rc), rc );
 508                         Debug( LDAP_DEBUG_ANY,
 509                                 "=> mdb_tool_next_id: %s\n", text->bv_val, 0, 0 );
 510                 } else if ( hole ) {
 511                         MDB_val key, data;
 512                         if ( nholes == nhmax - 1 ) {
 513                                 if ( holes == hbuf ) {
 514                                         holes = ch_malloc( nhmax * sizeof(dn_id) * 2 );
 515                                         AC_MEMCPY( holes, hbuf, sizeof(hbuf) );
 516                                 } else {
 517                                         holes = ch_realloc( holes, nhmax * sizeof(dn_id) * 2 );
 518                                 }
 519                                 nhmax *= 2;
 520                         }
 521                         ber_dupbv( &holes[nholes].dn, &ndn );
 522                         holes[nholes++].id = e->e_id;
 523                         key.mv_size = sizeof(ID);
 524                         key.mv_data = &e->e_id;
 525                         data.mv_size = 0;
 526                         data.mv_data = NULL;
 527                         rc = mdb_cursor_put( idcursor, &key, &data, MDB_NOOVERWRITE );
 528                         if ( rc == MDB_KEYEXIST )
 529                                 rc = 0;
 530                         if ( rc ) {
 531                                 snprintf( text->bv_val, text->bv_len,
 532                                         "dummy id2entry add failed: %s (%d)",
 533                                         mdb_strerror(rc), rc );
 534                                 Debug( LDAP_DEBUG_ANY,
 535                                         "=> mdb_tool_next_id: %s\n", text->bv_val, 0, 0 );
 536                         }
 537                 }
 538         } else if ( !hole ) {
 539                 unsigned i, j;
 540
 541                 e->e_id = id;
 542
 543                 for ( i=0; i<nholes; i++) {
 544                         if ( holes[i].id == e->e_id ) {
 545                                 free(holes[i].dn.bv_val);
 546                                 for (j=i;j<nholes;j++) holes[j] = holes[j+1];
 547                                 holes[j].id = 0;
 548                                 nholes--;
 549                                 break;
 550                         } else if ( holes[i].id > e->e_id ) {
 551                                 break;
 552                         }
 553                 }
 554         }
 555         return rc;
 556 }
 557
 558 static int
 559 mdb_tool_index_add(
 560         Operation *op,
 561         MDB_txn *txn,
 562         Entry *e )
 563 {
 564         struct mdb_info *mdb = (struct mdb_info *) op->o_bd->be_private;
 565
 566         if ( !mdb->mi_nattrs )
 567                 return 0;
 568
 569         if ( mdb_tool_threads > 1 ) {
 570                 IndexRec *ir;
 571                 int i, rc;
 572                 Attribute *a;
 573
 574                 ir = mdb_tool_index_rec;
 575                 for (i=0; i<mdb->mi_nattrs; i++)
 576                         ir[i].ir_attrs = NULL;
 577
 578                 for ( a = e->e_attrs; a != NULL; a = a->a_next ) {
 579                         rc = mdb_index_recset( mdb, a, a->a_desc->ad_type,
 580                                 &a->a_desc->ad_tags, ir );
 581                         if ( rc )
 582                                 return rc;
 583                 }
 584                 for (i=0; i<mdb->mi_nattrs; i++) {
 585                         if ( !ir[i].ir_ai )
 586                                 break;
 587                         rc = mdb_cursor_open( txn, ir[i].ir_ai->ai_dbi,
 588                                  &ir[i].ir_ai->ai_cursor );
 589                         if ( rc )
 590                                 return rc;
 591                 }
 592                 mdb_tool_ix_id = e->e_id;
 593                 mdb_tool_ix_txn = txn;
 594                 ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex );
 595                 /* Wait for all threads to be ready */
 596                 while ( mdb_tool_index_tcount ) {
 597                         ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main,
 598                                 &mdb_tool_index_mutex );
 599                 }
 600
 601                 for ( i=1; i<mdb_tool_threads; i++ )
 602                         mdb_tool_index_rec[i].ir_i = LDAP_BUSY;
 603                 mdb_tool_index_tcount = mdb_tool_threads - 1;
 604                 ldap_pvt_thread_cond_broadcast( &mdb_tool_index_cond_work );
 605                 ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex );
 606
 607                 return mdb_index_recrun( op, txn, mdb, ir, e->e_id, 0 );
 608         } else
 609         {
 610                 return mdb_index_entry_add( op, txn, e );
 611         }
 612 }
 613
 614 static int
 615 mdb_tool_index_finish()
 616 {
 617         int i, rc = 0;
 618         ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex );
 619         for ( i=1; i<mdb_tool_threads; i++ ) {
 620                 if ( mdb_tool_index_rec[i].ir_i == LDAP_BUSY ) {
 621                         ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main,
 622                                 &mdb_tool_index_mutex );
 623                         i--;
 624                         continue;
 625                 }
 626                 if ( mdb_tool_index_rec[i].ir_i ) {
 627                         rc = mdb_tool_index_rec[i].ir_i;
 628                         break;
 629                 }
 630         }
 631         ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex );
 632         return rc;
 633 }
 634
 635 ID mdb_tool_entry_put(
 636         BackendDB *be,
 637         Entry *e,
 638         struct berval *text )
 639 {
 640         int rc;
 641         struct mdb_info *mdb;
 642         Operation op = {0};
 643         Opheader ohdr = {0};
 644
 645         assert( be != NULL );
 646         assert( slapMode & SLAP_TOOL_MODE );
 647
 648         assert( text != NULL );
 649         assert( text->bv_val != NULL );
 650         assert( text->bv_val[0] == '\0' );      /* overconservative? */
 651
 652         Debug( LDAP_DEBUG_TRACE, "=> " LDAP_XSTRING(mdb_tool_entry_put)
 653                 "( %ld, \"%s\" )\n", (long) e->e_id, e->e_dn, 0 );
 654
 655         mdb = (struct mdb_info *) be->be_private;
 656
 657         if ( !mdb_tool_txn ) {
 658                 rc = mdb_txn_begin( mdb->mi_dbenv, NULL, 0, &mdb_tool_txn );
 659                 if( rc != 0 ) {
 660                         snprintf( text->bv_val, text->bv_len,
 661                                 "txn_begin failed: %s (%d)",
 662                                 mdb_strerror(rc), rc );
 663                         Debug( LDAP_DEBUG_ANY,
 664                                 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
 665                                  text->bv_val, 0, 0 );
 666                         return NOID;
 667                 }
 668                 rc = mdb_cursor_open( mdb_tool_txn, mdb->mi_id2entry, &idcursor );
 669                 if( rc != 0 ) {
 670                         snprintf( text->bv_val, text->bv_len,
 671                                 "cursor_open failed: %s (%d)",
 672                                 mdb_strerror(rc), rc );
 673                         Debug( LDAP_DEBUG_ANY,
 674                                 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
 675                                  text->bv_val, 0, 0 );
 676                         return NOID;
 677                 }
 678                 if ( !mdb->mi_nextid ) {
 679                         ID dummy;
 680                         mdb_next_id( be, idcursor, &dummy );
 681                 }
 682                 rc = mdb_cursor_open( mdb_tool_txn, mdb->mi_dn2id, &mcp );
 683                 if( rc != 0 ) {
 684                         snprintf( text->bv_val, text->bv_len,
 685                                 "cursor_open failed: %s (%d)",
 686                                 mdb_strerror(rc), rc );
 687                         Debug( LDAP_DEBUG_ANY,
 688                                 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
 689                                  text->bv_val, 0, 0 );
 690                         return NOID;
 691                 }
 692                 rc = mdb_cursor_open( mdb_tool_txn, mdb->mi_dn2id, &mcd );
 693                 if( rc != 0 ) {
 694                         snprintf( text->bv_val, text->bv_len,
 695                                 "cursor_open failed: %s (%d)",
 696                                 mdb_strerror(rc), rc );
 697                         Debug( LDAP_DEBUG_ANY,
 698                                 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
 699                                  text->bv_val, 0, 0 );
 700                         return NOID;
 701                 }
 702         }
 703
 704         op.o_hdr = &ohdr;
 705         op.o_bd = be;
 706         op.o_tmpmemctx = NULL;
 707         op.o_tmpmfuncs = &ch_mfuncs;
 708
 709         /* add dn2id indices */
 710         rc = mdb_tool_next_id( &op, mdb_tool_txn, e, text, 0 );
 711         if( rc != 0 ) {
 712                 goto done;
 713         }
 714
 715         if ( mdb_tool_threads > 1 ) {
 716                 LDAP_SLIST_INSERT_HEAD( &op.o_extra, &mdb_tool_axinfo[0]->ai_oe, oe_next );
 717         }
 718         rc = mdb_tool_index_add( &op, mdb_tool_txn, e );
 719         if( rc != 0 ) {
 720                 snprintf( text->bv_val, text->bv_len,
 721                                 "index_entry_add failed: err=%d", rc );
 722                 Debug( LDAP_DEBUG_ANY,
 723                         "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
 724                         text->bv_val, 0, 0 );
 725                 goto done;
 726         }
 727
 728
 729         /* id2entry index */
 730         rc = mdb_id2entry_add( &op, mdb_tool_txn, idcursor, e );
 731         if( rc != 0 ) {
 732                 snprintf( text->bv_val, text->bv_len,
 733                                 "id2entry_add failed: err=%d", rc );
 734                 Debug( LDAP_DEBUG_ANY,
 735                         "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
 736                         text->bv_val, 0, 0 );
 737                 goto done;
 738         }
 739
 740         if( mdb->mi_nattrs && mdb_tool_threads > 1 )
 741                 rc = mdb_tool_index_finish();
 742
 743 done:
 744         if( rc == 0 ) {
 745                 mdb_writes++;
 746                 if ( mdb_writes >= mdb_writes_per_commit ) {
 747                         unsigned i;
 748                         MDB_TOOL_IDL_FLUSH( be, mdb_tool_txn );
 749                         rc = mdb_txn_commit( mdb_tool_txn );
 750                         for ( i=0; i<mdb->mi_nattrs; i++ )
 751                                 mdb->mi_attrs[i]->ai_cursor = NULL;
 752                         mdb_writes = 0;
 753                         mdb_tool_txn = NULL;
 754                         idcursor = NULL;
 755                         if( rc != 0 ) {
 756                                 mdb->mi_numads = 0;
 757                                 snprintf( text->bv_val, text->bv_len,
 758                                                 "txn_commit failed: %s (%d)",
 759                                                 mdb_strerror(rc), rc );
 760                                 Debug( LDAP_DEBUG_ANY,
 761                                         "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
 762                                         text->bv_val, 0, 0 );
 763                                 e->e_id = NOID;
 764                         }
 765                 }
 766
 767         } else {
 768                 unsigned i;
 769                 mdb_txn_abort( mdb_tool_txn );
 770                 mdb_tool_txn = NULL;
 771                 idcursor = NULL;
 772                 for ( i=0; i<mdb->mi_nattrs; i++ )
 773                         mdb->mi_attrs[i]->ai_cursor = NULL;
 774                 mdb_writes = 0;
 775                 snprintf( text->bv_val, text->bv_len,
 776                         "txn_aborted! %s (%d)",
 777                         rc == LDAP_OTHER ? "Internal error" :
 778                         mdb_strerror(rc), rc );
 779                 Debug( LDAP_DEBUG_ANY,
 780                         "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
 781                         text->bv_val, 0, 0 );
 782                 e->e_id = NOID;
 783         }
 784
 785         return e->e_id;
 786 }
 787
 788 static int mdb_dn2id_upgrade( BackendDB *be );
 789
 790 int mdb_tool_entry_reindex(
 791         BackendDB *be,
 792         ID id,
 793         AttributeDescription **adv )
 794 {
 795         struct mdb_info *mi = (struct mdb_info *) be->be_private;
 796         int rc;
 797         Entry *e;
 798         Operation op = {0};
 799         Opheader ohdr = {0};
 800
 801         Debug( LDAP_DEBUG_ARGS,
 802                 "=> " LDAP_XSTRING(mdb_tool_entry_reindex) "( %ld )\n",
 803                 (long) id, 0, 0 );
 804         assert( tool_base == NULL );
 805         assert( tool_filter == NULL );
 806
 807         /* Special: do a dn2id upgrade */
 808         if ( adv && adv[0] == slap_schema.si_ad_entryDN ) {
 809                 /* short-circuit tool_entry_next() */
 810                 mdb_cursor_get( cursor, &key, &data, MDB_LAST );
 811                 return mdb_dn2id_upgrade( be );
 812         }
 813
 814         /* No indexes configured, nothing to do. Could return an
 815          * error here to shortcut things.
 816          */
 817         if (!mi->mi_attrs) {
 818                 return 0;
 819         }
 820
 821         /* Check for explicit list of attrs to index */
 822         if ( adv ) {
 823                 int i, j, n;
 824
 825                 if ( mi->mi_attrs[0]->ai_desc != adv[0] ) {
 826                         /* count */
 827                         for ( n = 0; adv[n]; n++ ) ;
 828
 829                         /* insertion sort */
 830                         for ( i = 0; i < n; i++ ) {
 831                                 AttributeDescription *ad = adv[i];
 832                                 for ( j = i-1; j>=0; j--) {
 833                                         if ( SLAP_PTRCMP( adv[j], ad ) <= 0 ) break;
 834                                         adv[j+1] = adv[j];
 835                                 }
 836                                 adv[j+1] = ad;
 837                         }
 838                 }
 839
 840                 for ( i = 0; adv[i]; i++ ) {
 841                         if ( mi->mi_attrs[i]->ai_desc != adv[i] ) {
 842                                 for ( j = i+1; j < mi->mi_nattrs; j++ ) {
 843                                         if ( mi->mi_attrs[j]->ai_desc == adv[i] ) {
 844                                                 AttrInfo *ai = mi->mi_attrs[i];
 845                                                 mi->mi_attrs[i] = mi->mi_attrs[j];
 846                                                 mi->mi_attrs[j] = ai;
 847                                                 break;
 848                                         }
 849                                 }
 850                                 if ( j == mi->mi_nattrs ) {
 851                                         Debug( LDAP_DEBUG_ANY,
 852                                                 LDAP_XSTRING(mdb_tool_entry_reindex)
 853                                                 ": no index configured for %s\n",
 854                                                 adv[i]->ad_cname.bv_val, 0, 0 );
 855                                         return -1;
 856                                 }
 857                         }
 858                 }
 859                 mi->mi_nattrs = i;
 860         }
 861
 862         e = mdb_tool_entry_get( be, id );
 863
 864         if( e == NULL ) {
 865                 Debug( LDAP_DEBUG_ANY,
 866                         LDAP_XSTRING(mdb_tool_entry_reindex)
 867                         ": could not locate id=%ld\n",
 868                         (long) id, 0, 0 );
 869                 return -1;
 870         }
 871
 872         if ( !txi ) {
 873                 rc = mdb_txn_begin( mi->mi_dbenv, NULL, 0, &txi );
 874                 if( rc != 0 ) {
 875                         Debug( LDAP_DEBUG_ANY,
 876                                 "=> " LDAP_XSTRING(mdb_tool_entry_reindex) ": "
 877                                 "txn_begin failed: %s (%d)\n",
 878                                 mdb_strerror(rc), rc, 0 );
 879                         goto done;
 880                 }
 881         }
 882
 883         if ( slapMode & SLAP_TRUNCATE_MODE ) {
 884                 int i;
 885                 for ( i=0; i < mi->mi_nattrs; i++ ) {
 886                         rc = mdb_drop( txi, mi->mi_attrs[i]->ai_dbi, 0 );
 887                         if ( rc ) {
 888                                 Debug( LDAP_DEBUG_ANY,
 889                                         LDAP_XSTRING(mdb_tool_entry_reindex)
 890                                         ": (Truncate) mdb_drop(%s) failed: %s (%d)\n",
 891                                         mi->mi_attrs[i]->ai_desc->ad_type->sat_cname.bv_val,
 892                                         mdb_strerror(rc), rc );
 893                                 return -1;
 894                         }
 895                 }
 896                 slapMode ^= SLAP_TRUNCATE_MODE;
 897         }
 898
 899         /*
 900          * just (re)add them for now
 901          * Use truncate mode to empty/reset index databases
 902          */
 903
 904         Debug( LDAP_DEBUG_TRACE,
 905                 "=> " LDAP_XSTRING(mdb_tool_entry_reindex) "( %ld )\n",
 906                 (long) id, 0, 0 );
 907
 908         op.o_hdr = &ohdr;
 909         op.o_bd = be;
 910         op.o_tmpmemctx = NULL;
 911         op.o_tmpmfuncs = &ch_mfuncs;
 912
 913         rc = mdb_tool_index_add( &op, txi, e );
 914
 915 done:
 916         if( rc == 0 ) {
 917                 mdb_writes++;
 918                 if ( mdb_writes >= mdb_writes_per_commit ) {
 919                         MDB_val key;
 920                         unsigned i;
 921                         MDB_TOOL_IDL_FLUSH( be, txi );
 922                         rc = mdb_txn_commit( txi );
 923                         mdb_writes = 0;
 924                         for ( i=0; i<mi->mi_nattrs; i++ )
 925                                 mi->mi_attrs[i]->ai_cursor = NULL;
 926                         if( rc != 0 ) {
 927                                 Debug( LDAP_DEBUG_ANY,
 928                                         "=> " LDAP_XSTRING(mdb_tool_entry_reindex)
 929                                         ": txn_commit failed: %s (%d)\n",
 930                                         mdb_strerror(rc), rc, 0 );
 931                                 e->e_id = NOID;
 932                         }
 933                         mdb_cursor_close( cursor );
 934                         txi = NULL;
 935                         /* Must close the read txn to allow old pages to be reclaimed. */
 936                         mdb_txn_abort( mdb_tool_txn );
 937                         /* and then reopen it so that tool_entry_next still works. */
 938                         mdb_txn_begin( mi->mi_dbenv, NULL, MDB_RDONLY, &mdb_tool_txn );
 939                         mdb_cursor_open( mdb_tool_txn, mi->mi_id2entry, &cursor );
 940                         key.mv_data = &id;
 941                         key.mv_size = sizeof(ID);
 942                         mdb_cursor_get( cursor, &key, NULL, MDB_SET );
 943                 }
 944
 945         } else {
 946                 unsigned i;
 947                 mdb_writes = 0;
 948                 mdb_cursor_close( cursor );
 949                 cursor = NULL;
 950                 mdb_txn_abort( txi );
 951                 for ( i=0; i<mi->mi_nattrs; i++ )
 952                         mi->mi_attrs[i]->ai_cursor = NULL;
 953                 Debug( LDAP_DEBUG_ANY,
 954                         "=> " LDAP_XSTRING(mdb_tool_entry_reindex)
 955                         ": txn_aborted! err=%d\n",
 956                         rc, 0, 0 );
 957                 e->e_id = NOID;
 958                 txi = NULL;
 959         }
 960         mdb_entry_release( &op, e, 0 );
 961
 962         return rc;
 963 }
 964
 965 ID mdb_tool_entry_modify(
 966         BackendDB *be,
 967         Entry *e,
 968         struct berval *text )
 969 {
 970         int rc;
 971         struct mdb_info *mdb;
 972         Operation op = {0};
 973         Opheader ohdr = {0};
 974
 975         assert( be != NULL );
 976         assert( slapMode & SLAP_TOOL_MODE );
 977
 978         assert( text != NULL );
 979         assert( text->bv_val != NULL );
 980         assert( text->bv_val[0] == '\0' );      /* overconservative? */
 981
 982         assert ( e->e_id != NOID );
 983
 984         Debug( LDAP_DEBUG_TRACE,
 985                 "=> " LDAP_XSTRING(mdb_tool_entry_modify) "( %ld, \"%s\" )\n",
 986                 (long) e->e_id, e->e_dn, 0 );
 987
 988         mdb = (struct mdb_info *) be->be_private;
 989
 990         if( cursor ) {
 991                 mdb_cursor_close( cursor );
 992                 cursor = NULL;
 993         }
 994         if ( !mdb_tool_txn ) {
 995                 rc = mdb_txn_begin( mdb->mi_dbenv, NULL, 0, &mdb_tool_txn );
 996                 if( rc != 0 ) {
 997                         snprintf( text->bv_val, text->bv_len,
 998                                 "txn_begin failed: %s (%d)",
 999                                 mdb_strerror(rc), rc );
1000                         Debug( LDAP_DEBUG_ANY,
1001                                 "=> " LDAP_XSTRING(mdb_tool_entry_modify) ": %s\n",
1002                                  text->bv_val, 0, 0 );
1003                         return NOID;
1004                 }
1005         }
1006
1007         op.o_hdr = &ohdr;
1008         op.o_bd = be;
1009         op.o_tmpmemctx = NULL;
1010         op.o_tmpmfuncs = &ch_mfuncs;
1011
1012         /* id2entry index */
1013         rc = mdb_id2entry_update( &op, mdb_tool_txn, NULL, e );
1014         if( rc != 0 ) {
1015                 snprintf( text->bv_val, text->bv_len,
1016                                 "id2entry_update failed: err=%d", rc );
1017                 Debug( LDAP_DEBUG_ANY,
1018                         "=> " LDAP_XSTRING(mdb_tool_entry_modify) ": %s\n",
1019                         text->bv_val, 0, 0 );
1020                 goto done;
1021         }
1022
1023 done:
1024         if( rc == 0 ) {
1025                 rc = mdb_txn_commit( mdb_tool_txn );
1026                 if( rc != 0 ) {
1027                         mdb->mi_numads = 0;
1028                         snprintf( text->bv_val, text->bv_len,
1029                                         "txn_commit failed: %s (%d)",
1030                                         mdb_strerror(rc), rc );
1031                         Debug( LDAP_DEBUG_ANY,
1032                                 "=> " LDAP_XSTRING(mdb_tool_entry_modify) ": "
1033                                 "%s\n", text->bv_val, 0, 0 );
1034                         e->e_id = NOID;
1035                 }
1036
1037         } else {
1038                 mdb_txn_abort( mdb_tool_txn );
1039                 snprintf( text->bv_val, text->bv_len,
1040                         "txn_aborted! %s (%d)",
1041                         mdb_strerror(rc), rc );
1042                 Debug( LDAP_DEBUG_ANY,
1043                         "=> " LDAP_XSTRING(mdb_tool_entry_modify) ": %s\n",
1044                         text->bv_val, 0, 0 );
1045                 e->e_id = NOID;
1046         }
1047         mdb_tool_txn = NULL;
1048
1049         return e->e_id;
1050 }
1051
1052 int mdb_tool_entry_delete(
1053         BackendDB *be,
1054         struct berval *ndn,
1055         struct berval *text )
1056 {
1057         int rc;
1058         struct mdb_info *mdb;
1059         Operation op = {0};
1060         Opheader ohdr = {0};
1061         Entry *e;
1062
1063         assert( be != NULL );
1064         assert( slapMode & SLAP_TOOL_MODE );
1065
1066         assert( text != NULL );
1067         assert( text->bv_val != NULL );
1068         assert( text->bv_val[0] == '\0' );      /* overconservative? */
1069
1070         assert ( ndn != NULL );
1071         assert ( ndn->bv_val != NULL );
1072
1073         Debug( LDAP_DEBUG_TRACE,
1074                 "=> " LDAP_XSTRING(mdb_tool_entry_delete) "( %s )\n",
1075                 ndn->bv_val, 0, 0 );
1076
1077         mdb = (struct mdb_info *) be->be_private;
1078
1079         assert( cursor == NULL );
1080         if( cursor ) {
1081                 mdb_cursor_close( cursor );
1082                 cursor = NULL;
1083         }
1084         if( !mdb_tool_txn ) {
1085                 rc = mdb_txn_begin( mdb->mi_dbenv, NULL, 0, &mdb_tool_txn );
1086                 if( rc != 0 ) {
1087                         snprintf( text->bv_val, text->bv_len,
1088                                 "txn_begin failed: %s (%d)",
1089                                 mdb_strerror(rc), rc );
1090                         Debug( LDAP_DEBUG_ANY,
1091                                 "=> " LDAP_XSTRING(mdb_tool_entry_delete) ": %s\n",
1092                                  text->bv_val, 0, 0 );
1093                         return LDAP_OTHER;
1094                 }
1095         }
1096
1097         rc = mdb_cursor_open( mdb_tool_txn, mdb->mi_dn2id, &cursor );
1098         if( rc != 0 ) {
1099                 snprintf( text->bv_val, text->bv_len,
1100                         "cursor_open failed: %s (%d)",
1101                         mdb_strerror(rc), rc );
1102                 Debug( LDAP_DEBUG_ANY,
1103                         "=> " LDAP_XSTRING(mdb_tool_entry_delete) ": %s\n",
1104                          text->bv_val, 0, 0 );
1105                 return LDAP_OTHER;
1106         }
1107
1108         op.o_hdr = &ohdr;
1109         op.o_bd = be;
1110         op.o_tmpmemctx = NULL;
1111         op.o_tmpmfuncs = &ch_mfuncs;
1112
1113         rc = mdb_dn2entry( &op, mdb_tool_txn, cursor, ndn, &e, NULL, 0 );
1114         if( rc != 0 ) {
1115                 snprintf( text->bv_val, text->bv_len,
1116                         "dn2entry failed: %s (%d)",
1117                         mdb_strerror(rc), rc );
1118                 Debug( LDAP_DEBUG_ANY,
1119                         "=> " LDAP_XSTRING(mdb_tool_entry_delete) ": %s\n",
1120                          text->bv_val, 0, 0 );
1121                 goto done;
1122         }
1123
1124         /* check that we wouldn't orphan any children */
1125         rc = mdb_dn2id_children( &op, mdb_tool_txn, e );
1126         if( rc != MDB_NOTFOUND ) {
1127                 switch( rc ) {
1128                 case 0:
1129                         snprintf( text->bv_val, text->bv_len,
1130                                 "delete failed:"
1131                                 " subordinate objects must be deleted first");
1132                         break;
1133                 default:
1134                         snprintf( text->bv_val, text->bv_len,
1135                                 "has_children failed: %s (%d)",
1136                                 mdb_strerror(rc), rc );
1137                         break;
1138                 }
1139                 rc = -1;
1140                 Debug( LDAP_DEBUG_ANY,
1141                         "=> " LDAP_XSTRING(mdb_tool_entry_delete) ": %s\n",
1142                          text->bv_val, 0, 0 );
1143                 goto done;
1144         }
1145
1146         /* delete from dn2id */
1147         rc = mdb_dn2id_delete( &op, cursor, e->e_id, 1 );
1148         if( rc != 0 ) {
1149                 snprintf( text->bv_val, text->bv_len,
1150                                 "dn2id_delete failed: err=%d", rc );
1151                 Debug( LDAP_DEBUG_ANY,
1152                         "=> " LDAP_XSTRING(mdb_tool_entry_delete) ": %s\n",
1153                         text->bv_val, 0, 0 );
1154                 goto done;
1155         }
1156
1157         /* deindex values */
1158         rc = mdb_index_entry_del( &op, mdb_tool_txn, e );
1159         if( rc != 0 ) {
1160                 snprintf( text->bv_val, text->bv_len,
1161                                 "entry_delete failed: err=%d", rc );
1162                 Debug( LDAP_DEBUG_ANY,
1163                         "=> " LDAP_XSTRING(mdb_tool_entry_delete) ": %s\n",
1164                         text->bv_val, 0, 0 );
1165                 goto done;
1166         }
1167
1168         /* do the deletion */
1169         rc = mdb_id2entry_delete( be, mdb_tool_txn, e );
1170         if( rc != 0 ) {
1171                 snprintf( text->bv_val, text->bv_len,
1172                                 "id2entry_update failed: err=%d", rc );
1173                 Debug( LDAP_DEBUG_ANY,
1174                         "=> " LDAP_XSTRING(mdb_tool_entry_delete) ": %s\n",
1175                         text->bv_val, 0, 0 );
1176                 goto done;
1177         }
1178
1179 done:
1180         /* free entry */
1181         if( e != NULL ) {
1182                 mdb_entry_return( &op, e );
1183         }
1184
1185         if( rc == 0 ) {
1186                 rc = mdb_txn_commit( mdb_tool_txn );
1187                 if( rc != 0 ) {
1188                         snprintf( text->bv_val, text->bv_len,
1189                                         "txn_commit failed: %s (%d)",
1190                                         mdb_strerror(rc), rc );
1191                         Debug( LDAP_DEBUG_ANY,
1192                                 "=> " LDAP_XSTRING(mdb_tool_entry_delete) ": "
1193                                 "%s\n", text->bv_val, 0, 0 );
1194                 }
1195
1196         } else {
1197                 mdb_txn_abort( mdb_tool_txn );
1198                 snprintf( text->bv_val, text->bv_len,
1199                         "txn_aborted! %s (%d)",
1200                         mdb_strerror(rc), rc );
1201                 Debug( LDAP_DEBUG_ANY,
1202                         "=> " LDAP_XSTRING(mdb_tool_entry_delete) ": %s\n",
1203                         text->bv_val, 0, 0 );
1204         }
1205         mdb_tool_txn = NULL;
1206         cursor = NULL;
1207
1208         return rc;
1209 }
1210
1211 static void *
1212 mdb_tool_index_task( void *ctx, void *ptr )
1213 {
1214         int base = *(int *)ptr;
1215         Operation op = {0};
1216         Opheader ohdr = {0};
1217         AttrIxInfo ai = {0}, *aio;
1218
1219         free( ptr );
1220         op.o_hdr = &ohdr;
1221         op.o_bd = mdb_tool_ix_be;
1222         op.o_tmpmemctx = NULL;
1223         op.o_tmpmfuncs = &ch_mfuncs;
1224         aio = mdb_tool_axinfo[base];
1225         mdb_tool_axinfo[base] = &ai;
1226         LDAP_SLIST_INSERT_HEAD( &op.o_extra, &ai.ai_oe, oe_next );
1227         while ( 1 ) {
1228                 ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex );
1229                 mdb_tool_index_tcount--;
1230                 if ( !mdb_tool_index_tcount )
1231                         ldap_pvt_thread_cond_signal( &mdb_tool_index_cond_main );
1232                 ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_work,
1233                         &mdb_tool_index_mutex );
1234                 if ( slapd_shutdown ) {
1235                         mdb_tool_index_tcount--;
1236                         if ( !mdb_tool_index_tcount )
1237                                 ldap_pvt_thread_cond_signal( &mdb_tool_index_cond_main );
1238                         *aio = ai;
1239                         mdb_tool_axinfo[base] = aio;
1240                         ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex );
1241                         break;
1242                 }
1243                 ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex );
1244                 mdb_tool_index_rec[base].ir_i = mdb_index_recrun( &op,
1245                         mdb_tool_ix_txn,
1246                         mdb_tool_info, mdb_tool_index_rec, mdb_tool_ix_id, base );
1247         }
1248
1249         return NULL;
1250 }
1251
1252 #ifdef MDB_TOOL_IDL_CACHING
1253 static int
1254 mdb_tool_idl_cmp( const void *v1, const void *v2 )
1255 {
1256         const mdb_tool_idl_cache *c1 = v1, *c2 = v2;
1257         int rc;
1258
1259         if (( rc = c1->kstr.bv_len - c2->kstr.bv_len )) return rc;
1260         return memcmp( c1->kstr.bv_val, c2->kstr.bv_val, c1->kstr.bv_len );
1261 }
1262
1263 static int
1264 mdb_tool_idl_flush_one( MDB_cursor *mc, AttrIxInfo *ai, mdb_tool_idl_cache *ic )
1265 {
1266         mdb_tool_idl_cache_entry *ice;
1267         MDB_val key, data[2];
1268         int i, rc;
1269         ID id, nid;
1270
1271         /* Freshly allocated, ignore it */
1272         if ( !ic->head && ic->count <= MDB_IDL_DB_SIZE ) {
1273                 return 0;
1274         }
1275
1276         key.mv_data = ic->kstr.bv_val;
1277         key.mv_size = ic->kstr.bv_len;
1278
1279         if ( ic->count > MDB_IDL_DB_SIZE ) {
1280                 while ( ic->flags & WAS_FOUND ) {
1281                         rc = mdb_cursor_get( mc, &key, data, MDB_SET );
1282                         if ( rc ) {
1283                                 /* FIXME: find out why this happens */
1284                                 ic->flags = 0;
1285                                 break;
1286                         }
1287                         if ( ic->flags & WAS_RANGE ) {
1288                                 /* Skip lo */
1289                                 rc = mdb_cursor_get( mc, &key, data, MDB_NEXT_DUP );
1290
1291                                 /* Get hi */
1292                                 rc = mdb_cursor_get( mc, &key, data, MDB_NEXT_DUP );
1293
1294                                 /* Store range hi */
1295                                 data[0].mv_data = &ic->last;
1296                                 rc = mdb_cursor_put( mc, &key, data, MDB_CURRENT );
1297                         } else {
1298                                 /* Delete old data, replace with range */
1299                                 ic->first = *(ID *)data[0].mv_data;
1300                                 mdb_cursor_del( mc, MDB_NODUPDATA );
1301                         }
1302                         break;
1303                 }
1304                 if ( !(ic->flags & WAS_RANGE)) {
1305                         /* range, didn't exist before */
1306                         nid = 0;
1307                         data[0].mv_size = sizeof(ID);
1308                         data[0].mv_data = &nid;
1309                         rc = mdb_cursor_put( mc, &key, data, 0 );
1310                         if ( rc == 0 ) {
1311                                 data[0].mv_data = &ic->first;
1312                                 rc = mdb_cursor_put( mc, &key, data, 0 );
1313                                 if ( rc == 0 ) {
1314                                         data[0].mv_data = &ic->last;
1315                                         rc = mdb_cursor_put( mc, &key, data, 0 );
1316                                 }
1317                         }
1318                         if ( rc ) {
1319                                 rc = -1;
1320                         }
1321                 }
1322         } else {
1323                 /* Normal write */
1324                 int n;
1325
1326                 data[0].mv_size = sizeof(ID);
1327                 rc = 0;
1328                 for ( ice = ic->head, n=0; ice; ice = ice->next, n++ ) {
1329                         int end;
1330                         if ( ice->next ) {
1331                                 end = IDBLOCK;
1332                         } else {
1333                                 end = (ic->count-ic->offset) & (IDBLOCK-1);
1334                                 if ( !end )
1335                                         end = IDBLOCK;
1336                         }
1337                         data[1].mv_size = end;
1338                         data[0].mv_data = ice->ids;
1339                         rc = mdb_cursor_put( mc, &key, data, MDB_APPENDDUP|MDB_MULTIPLE );
1340                         if ( rc ) {
1341                                 rc = -1;
1342                                 break;
1343                         }
1344                 }
1345                 if ( ic->head ) {
1346                         ic->tail->next = ai->ai_flist;
1347                         ai->ai_flist = ic->head;
1348                 }
1349         }
1350         ic->head = ai->ai_clist;
1351         ai->ai_clist = ic;
1352         return rc;
1353 }
1354
1355 static int
1356 mdb_tool_idl_flush_db( MDB_txn *txn, AttrInfo *ai, AttrIxInfo *ax )
1357 {
1358         MDB_cursor *mc;
1359         Avlnode *root;
1360         int rc;
1361
1362         mdb_cursor_open( txn, ai->ai_dbi, &mc );
1363         root = tavl_end( ai->ai_root, TAVL_DIR_LEFT );
1364         do {
1365                 rc = mdb_tool_idl_flush_one( mc, ax, root->avl_data );
1366                 if ( rc != -1 )
1367                         rc = 0;
1368         } while ((root = tavl_next(root, TAVL_DIR_RIGHT)));
1369         mdb_cursor_close( mc );
1370
1371         return rc;
1372 }
1373
1374 static int
1375 mdb_tool_idl_flush( BackendDB *be, MDB_txn *txn )
1376 {
1377         struct mdb_info *mdb = (struct mdb_info *) be->be_private;
1378         int rc = 0;
1379         unsigned int i, dbi;
1380
1381         for ( i=0; i < mdb->mi_nattrs; i++ ) {
1382                 if ( !mdb->mi_attrs[i]->ai_root ) continue;
1383                 rc = mdb_tool_idl_flush_db( txn, mdb->mi_attrs[i], mdb_tool_axinfo[i % mdb_tool_threads] );
1384                 tavl_free(mdb->mi_attrs[i]->ai_root, NULL);
1385                 mdb->mi_attrs[i]->ai_root = NULL;
1386                 if ( rc )
1387                         break;
1388         }
1389         return rc;
1390 }
1391
1392 int mdb_tool_idl_add(
1393         BackendDB *be,
1394         MDB_cursor *mc,
1395         struct berval *keys,
1396         ID id )
1397 {
1398         MDB_dbi dbi;
1399         mdb_tool_idl_cache *ic, itmp;
1400         mdb_tool_idl_cache_entry *ice;
1401         int i, rc, lcount;
1402         AttrIxInfo *ax = (AttrIxInfo *)mc;
1403         AttrInfo *ai = (AttrInfo *)ax->ai_ai;
1404         mc = ai->ai_cursor;
1405
1406         dbi = ai->ai_dbi;
1407         for (i=0; keys[i].bv_val; i++) {
1408         itmp.kstr = keys[i];
1409         ic = tavl_find( ai->ai_root, &itmp, mdb_tool_idl_cmp );
1410
1411         /* No entry yet, create one */
1412         if ( !ic ) {
1413                 MDB_val key, data;
1414                 ID nid;
1415                 int rc;
1416
1417                 if ( ax->ai_clist ) {
1418                         ic = ax->ai_clist;
1419                         ax->ai_clist = ic->head;
1420                 } else {
1421                         ic = ch_malloc( sizeof( mdb_tool_idl_cache ) + itmp.kstr.bv_len + 4 );
1422                 }
1423                 ic->kstr.bv_len = itmp.kstr.bv_len;
1424                 ic->kstr.bv_val = (char *)(ic+1);
1425                 memcpy( ic->kstr.bv_val, itmp.kstr.bv_val, ic->kstr.bv_len );
1426                 ic->head = ic->tail = NULL;
1427                 ic->last = 0;
1428                 ic->count = 0;
1429                 ic->offset = 0;
1430                 ic->flags = 0;
1431                 tavl_insert( &ai->ai_root, ic, mdb_tool_idl_cmp,
1432                         avl_dup_error );
1433
1434                 /* load existing key count here */
1435                 key.mv_size = keys[i].bv_len;
1436                 key.mv_data = keys[i].bv_val;
1437                 rc = mdb_cursor_get( mc, &key, &data, MDB_SET );
1438                 if ( rc == 0 ) {
1439                         ic->flags |= WAS_FOUND;
1440                         nid = *(ID *)data.mv_data;
1441                         if ( nid == 0 ) {
1442                                 ic->count = MDB_IDL_DB_SIZE+1;
1443                                 ic->flags |= WAS_RANGE;
1444                         } else {
1445                                 size_t count;
1446
1447                                 mdb_cursor_count( mc, &count );
1448                                 ic->count = count;
1449                                 ic->first = nid;
1450                                 ic->offset = count & (IDBLOCK-1);
1451                         }
1452                 }
1453         }
1454         /* are we a range already? */
1455         if ( ic->count > MDB_IDL_DB_SIZE ) {
1456                 ic->last = id;
1457                 continue;
1458         /* Are we at the limit, and converting to a range? */
1459         } else if ( ic->count == MDB_IDL_DB_SIZE ) {
1460                 if ( ic->head ) {
1461                         ic->tail->next = ax->ai_flist;
1462                         ax->ai_flist = ic->head;
1463                 }
1464                 ic->head = ic->tail = NULL;
1465                 ic->last = id;
1466                 ic->count++;
1467                 continue;
1468         }
1469         /* No free block, create that too */
1470         lcount = (ic->count-ic->offset) & (IDBLOCK-1);
1471         if ( !ic->tail || lcount == 0) {
1472                 if ( ax->ai_flist ) {
1473                         ice = ax->ai_flist;
1474                         ax->ai_flist = ice->next;
1475                 } else {
1476                         ice = ch_malloc( sizeof( mdb_tool_idl_cache_entry ));
1477                 }
1478                 ice->next = NULL;
1479                 if ( !ic->head ) {
1480                         ic->head = ice;
1481                 } else {
1482                         ic->tail->next = ice;
1483                 }
1484                 ic->tail = ice;
1485                 if ( lcount )
1486                         ice->ids[lcount-1] = 0;
1487                 if ( !ic->count )
1488                         ic->first = id;
1489         }
1490         ice = ic->tail;
1491         if (!lcount || ice->ids[lcount-1] != id) {
1492                 ice->ids[lcount] = id;
1493                 ic->count++;
1494         }
1495         }
1496
1497         return 0;
1498 }
1499 #endif /* MDB_TOOL_IDL_CACHING */
1500
1501 /* Upgrade from pre 2.4.34 dn2id format */
1502
1503 #include <ac/unistd.h>
1504 #include <lutil_meter.h>
1505
1506 #define STACKSIZ        2048
1507
1508 typedef struct rec {
1509         ID id;
1510         size_t len;
1511         char rdn[512];
1512 } rec;
1513
1514 static int
1515 mdb_dn2id_upgrade( BackendDB *be ) {
1516         struct mdb_info *mi = (struct mdb_info *) be->be_private;
1517         MDB_txn *mt;
1518         MDB_cursor *mc = NULL;
1519         MDB_val key, data;
1520         char *ptr;
1521         int rc, writes=0, depth=0;
1522         int enable_meter = 0;
1523         ID id = 0, *num, count = 0;
1524         rec *stack;
1525         lutil_meter_t meter;
1526
1527         if (!(mi->mi_flags & MDB_NEED_UPGRADE)) {
1528                 Debug( LDAP_DEBUG_ANY, "database %s: No upgrade needed.\n",
1529                         be->be_suffix[0].bv_val, 0, 0 );
1530                 return 0;
1531         }
1532
1533         {
1534                 MDB_stat st;
1535
1536                 mdb_stat(mdb_cursor_txn(cursor), mi->mi_dbis[MDB_ID2ENTRY], &st);
1537                 if (!st.ms_entries) {
1538                         /* Empty DB, nothing to upgrade? */
1539                         return 0;
1540                 }
1541                 if (isatty(2))
1542                         enable_meter = !lutil_meter_open(&meter,
1543                                 &lutil_meter_text_display,
1544                                 &lutil_meter_linear_estimator,
1545                                 st.ms_entries);
1546         }
1547
1548         num = ch_malloc(STACKSIZ * (sizeof(ID) + sizeof(rec)));
1549         stack = (rec *)(num + STACKSIZ);
1550
1551         rc = mdb_txn_begin(mi->mi_dbenv, NULL, 0, &mt);
1552         if (rc) {
1553                 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_begin failed, %s (%d)\n",
1554                         mdb_strerror(rc), rc, 0 );
1555                 goto leave;
1556         }
1557         rc = mdb_cursor_open(mt, mi->mi_dbis[MDB_DN2ID], &mc);
1558         if (rc) {
1559                 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_open failed, %s (%d)\n",
1560                         mdb_strerror(rc), rc, 0 );
1561                 goto leave;
1562         }
1563
1564         key.mv_size = sizeof(ID);
1565         /* post-order depth-first update */
1566         for(;;) {
1567                 size_t dkids;
1568                 unsigned char *ptr;
1569
1570                 /* visit */
1571                 key.mv_data = &id;
1572                 stack[depth].id = id;
1573                 rc = mdb_cursor_get(mc, &key, &data, MDB_SET);
1574                 if (rc) {
1575                         Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_get failed, %s (%d)\n",
1576                                 mdb_strerror(rc), rc, 0 );
1577                         goto leave;
1578                 }
1579                 num[depth] = 1;
1580
1581                 rc = mdb_cursor_count(mc, &dkids);
1582                 if (rc) {
1583                         Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_count failed, %s (%d)\n",
1584                                 mdb_strerror(rc), rc, 0 );
1585                         goto leave;
1586                 }
1587                 if (dkids > 1) {
1588                         rc = mdb_cursor_get(mc, &key, &data, MDB_NEXT_DUP);
1589 down:
1590                         ptr = (unsigned char *)data.mv_data + data.mv_size - sizeof(ID);
1591                         memcpy(&id, ptr, sizeof(ID));
1592                         depth++;
1593                         memcpy(stack[depth].rdn, data.mv_data, data.mv_size);
1594                         stack[depth].len = data.mv_size;
1595                         continue;
1596                 }
1597
1598
1599                 /* pop: write updated count, advance to next node */
1600 pop:
1601                 /* update superior counts */
1602                 if (depth)
1603                         num[depth-1] += num[depth];
1604
1605                 key.mv_data = &id;
1606                 id = stack[depth-1].id;
1607                 data.mv_data = stack[depth].rdn;
1608                 data.mv_size = stack[depth].len;
1609                 rc = mdb_cursor_get(mc, &key, &data, MDB_GET_BOTH);
1610                 if (rc) {
1611                         Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_get(BOTH) failed, %s (%d)\n",
1612                                 mdb_strerror(rc), rc, 0 );
1613                         goto leave;
1614                 }
1615                 data.mv_data = stack[depth].rdn;
1616                 ptr = (unsigned char *)data.mv_data + data.mv_size;
1617                 memcpy(ptr, &num[depth], sizeof(ID));
1618                 data.mv_size += sizeof(ID);
1619                 rc = mdb_cursor_del(mc, 0);
1620                 if (rc) {
1621                         Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_del failed, %s (%d)\n",
1622                                 mdb_strerror(rc), rc, 0 );
1623                         goto leave;
1624                 }
1625                 rc = mdb_cursor_put(mc, &key, &data, 0);
1626                 if (rc) {
1627                         Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_put failed, %s (%d)\n",
1628                                 mdb_strerror(rc), rc, 0 );
1629                         goto leave;
1630                 }
1631                 count++;
1632 #if 1
1633                 if (enable_meter)
1634                         lutil_meter_update(&meter, count, 0);
1635 #else
1636                 {
1637                         int len;
1638                         ptr = data.mv_data;
1639                         len = (ptr[0] & 0x7f) << 8 | ptr[1];
1640                         printf("ID: %zu, %zu, %.*s\n", stack[depth].id, num[depth], len, ptr+2);
1641                 }
1642 #endif
1643                 writes++;
1644                 if (writes == 1000) {
1645                         mdb_cursor_close(mc);
1646                         rc = mdb_txn_commit(mt);
1647                         if (rc) {
1648                                 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_commit failed, %s (%d)\n",
1649                                         mdb_strerror(rc), rc, 0 );
1650                                 goto leave;
1651                         }
1652                         rc = mdb_txn_begin(mi->mi_dbenv, NULL, 0, &mt);
1653                         if (rc) {
1654                                 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_begin(2) failed, %s (%d)\n",
1655                                         mdb_strerror(rc), rc, 0 );
1656                                 goto leave;
1657                         }
1658                         rc = mdb_cursor_open(mt, mi->mi_dbis[MDB_DN2ID], &mc);
1659                         if (rc) {
1660                                 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_open(2) failed, %s (%d)\n",
1661                                         mdb_strerror(rc), rc, 0 );
1662                                 goto leave;
1663                         }
1664                         rc = mdb_cursor_get(mc, &key, &data, MDB_GET_BOTH);
1665                         if (rc) {
1666                                 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_get(2) failed, %s (%d)\n",
1667                                         mdb_strerror(rc), rc, 0 );
1668                                 goto leave;
1669                         }
1670                         writes = 0;
1671                 }
1672                 depth--;
1673
1674                 rc = mdb_cursor_get(mc, &key, &data, MDB_NEXT_DUP);
1675                 if (rc == 0)
1676                         goto down;
1677                 rc = 0;
1678                 if (depth)
1679                         goto pop;
1680                 else
1681                         break;
1682         }
1683 leave:
1684         mdb_cursor_close(mc);
1685         if (mt) {
1686                 int r2;
1687                 r2 = mdb_txn_commit(mt);
1688                 if (r2) {
1689                         Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_commit(2) failed, %s (%d)\n",
1690                                 mdb_strerror(r2), r2, 0 );
1691                         if (!rc)
1692                                 rc = r2;
1693                 }
1694         }
1695         ch_free(num);
1696         if (enable_meter) {
1697                 lutil_meter_update(&meter, count, 1);
1698                 lutil_meter_close(&meter);
1699         }
1700         return rc;
1701 }