git.sur5r.net Git - openldap/blob - servers/slapd/back-mdb/tools.c

   1 /* tools.c - tools for slap tools */
   2 /* $OpenLDAP$ */
   3 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
   4  *
   5  * Copyright 2011-2014 The OpenLDAP Foundation.
   6  * All rights reserved.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted only as authorized by the OpenLDAP
  10  * Public License.
  11  *
  12  * A copy of this license is available in the file LICENSE in the
  13  * top-level directory of the distribution or, alternatively, at
  14  * <http://www.OpenLDAP.org/license.html>.
  15  */
  16
  17 #include "portable.h"
  18
  19 #include <stdio.h>
  20 #include <ac/string.h>
  21 #include <ac/errno.h>
  22
  23 #define AVL_INTERNAL
  24 #include "back-mdb.h"
  25 #include "idl.h"
  26
  27 #ifdef MDB_TOOL_IDL_CACHING
  28 static int mdb_tool_idl_flush( BackendDB *be, MDB_txn *txn );
  29
  30 #define IDBLOCK 1024
  31
  32 typedef struct mdb_tool_idl_cache_entry {
  33         struct mdb_tool_idl_cache_entry *next;
  34         ID ids[IDBLOCK];
  35 } mdb_tool_idl_cache_entry;
  36
  37 typedef struct mdb_tool_idl_cache {
  38         struct berval kstr;
  39         mdb_tool_idl_cache_entry *head, *tail;
  40         ID first, last;
  41         int count;
  42         short offset;
  43         short flags;
  44 } mdb_tool_idl_cache;
  45 #define WAS_FOUND       0x01
  46 #define WAS_RANGE       0x02
  47
  48 #define MDB_TOOL_IDL_FLUSH(be, txn)     mdb_tool_idl_flush(be, txn)
  49 #else
  50 #define MDB_TOOL_IDL_FLUSH(be, txn)
  51 #endif /* MDB_TOOL_IDL_CACHING */
  52
  53 MDB_txn *mdb_tool_txn = NULL;
  54
  55 static MDB_txn *txi = NULL;
  56 static MDB_cursor *cursor = NULL, *idcursor = NULL;
  57 static MDB_cursor *mcp = NULL, *mcd = NULL;
  58 static MDB_val key, data;
  59 static ID previd = NOID;
  60
  61 typedef struct dn_id {
  62         ID id;
  63         struct berval dn;
  64 } dn_id;
  65
  66 #define HOLE_SIZE       4096
  67 static dn_id hbuf[HOLE_SIZE], *holes = hbuf;
  68 static unsigned nhmax = HOLE_SIZE;
  69 static unsigned nholes;
  70
  71 static struct berval    *tool_base;
  72 static int              tool_scope;
  73 static Filter           *tool_filter;
  74 static Entry            *tool_next_entry;
  75
  76 static ID mdb_tool_ix_id;
  77 static BackendDB *mdb_tool_ix_be;
  78 static MDB_txn *mdb_tool_ix_txn;
  79 static int mdb_tool_index_tcount, mdb_tool_threads;
  80 static IndexRec *mdb_tool_index_rec;
  81 static AttrIxInfo **mdb_tool_axinfo;
  82 static struct mdb_info *mdb_tool_info;
  83 static ldap_pvt_thread_mutex_t mdb_tool_index_mutex;
  84 static ldap_pvt_thread_cond_t mdb_tool_index_cond_main;
  85 static ldap_pvt_thread_cond_t mdb_tool_index_cond_work;
  86 static void * mdb_tool_index_task( void *ctx, void *ptr );
  87
  88 static int      mdb_writes, mdb_writes_per_commit;
  89
  90 /* Number of ops per commit in Quick mode.
  91  * Batching speeds writes overall, but too large a
  92  * batch will fail with MDB_TXN_FULL.
  93  */
  94 #ifndef MDB_WRITES_PER_COMMIT
  95 #define MDB_WRITES_PER_COMMIT   500
  96 #endif
  97
  98 static int
  99 mdb_tool_entry_get_int( BackendDB *be, ID id, Entry **ep );
 100
 101 int mdb_tool_entry_open(
 102         BackendDB *be, int mode )
 103 {
 104         /* In Quick mode, commit once per 500 entries */
 105         mdb_writes = 0;
 106         if ( slapMode & SLAP_TOOL_QUICK )
 107                 mdb_writes_per_commit = MDB_WRITES_PER_COMMIT;
 108         else
 109                 mdb_writes_per_commit = 1;
 110
 111 #ifdef MDB_TOOL_IDL_CACHING                     /* threaded indexing has no performance advantage */
 112         /* Set up for threaded slapindex */
 113         if (( slapMode & (SLAP_TOOL_QUICK|SLAP_TOOL_READONLY)) == SLAP_TOOL_QUICK ) {
 114                 if ( !mdb_tool_info ) {
 115                         struct mdb_info *mdb = (struct mdb_info *) be->be_private;
 116                         ldap_pvt_thread_mutex_init( &mdb_tool_index_mutex );
 117                         ldap_pvt_thread_cond_init( &mdb_tool_index_cond_main );
 118                         ldap_pvt_thread_cond_init( &mdb_tool_index_cond_work );
 119                         if ( mdb->mi_nattrs ) {
 120                                 int i;
 121                                 mdb_tool_threads = slap_tool_thread_max - 1;
 122                                 if ( mdb_tool_threads > 1 ) {
 123                                         mdb_tool_index_rec = ch_calloc( mdb->mi_nattrs, sizeof( IndexRec ));
 124                                         mdb_tool_axinfo = ch_calloc( mdb_tool_threads, sizeof( AttrIxInfo* ) +
 125                                                 sizeof( AttrIxInfo ));
 126                                         mdb_tool_axinfo[0] = (AttrIxInfo *)(mdb_tool_axinfo + mdb_tool_threads);
 127                                         for (i=1; i<mdb_tool_threads; i++)
 128                                                 mdb_tool_axinfo[i] = mdb_tool_axinfo[i-1]+1;
 129                                         mdb_tool_index_tcount = mdb_tool_threads - 1;
 130                                         mdb_tool_ix_be = be;
 131                                         for (i=1; i<mdb_tool_threads; i++) {
 132                                                 int *ptr = ch_malloc( sizeof( int ));
 133                                                 *ptr = i;
 134                                                 ldap_pvt_thread_pool_submit( &connection_pool,
 135                                                         mdb_tool_index_task, ptr );
 136                                         }
 137                                         mdb_tool_info = mdb;
 138                                 }
 139                         }
 140                 }
 141         }
 142 #endif
 143
 144         return 0;
 145 }
 146
 147 int mdb_tool_entry_close(
 148         BackendDB *be )
 149 {
 150 #ifdef MDB_TOOL_IDL_CACHING
 151         if ( mdb_tool_info ) {
 152                 int i;
 153                 slapd_shutdown = 1;
 154                 ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex );
 155
 156                 /* There might still be some threads starting */
 157                 while ( mdb_tool_index_tcount > 0 ) {
 158                         ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main,
 159                                         &mdb_tool_index_mutex );
 160                 }
 161
 162                 mdb_tool_index_tcount = mdb_tool_threads - 1;
 163                 ldap_pvt_thread_cond_broadcast( &mdb_tool_index_cond_work );
 164
 165                 /* Make sure all threads are stopped */
 166                 while ( mdb_tool_index_tcount > 0 ) {
 167                         ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main,
 168                                 &mdb_tool_index_mutex );
 169                 }
 170                 ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex );
 171
 172                 mdb_tool_info = NULL;
 173                 slapd_shutdown = 0;
 174                 ch_free( mdb_tool_index_rec );
 175                 mdb_tool_index_tcount = mdb_tool_threads - 1;
 176                 if (txn)
 177                         MDB_TOOL_IDL_FLUSH( be, txn );
 178                 for (i=0; i<mdb_tool_threads; i++) {
 179                         mdb_tool_idl_cache *ic;
 180                         mdb_tool_idl_cache_entry *ice;
 181                         while ((ic = mdb_tool_axinfo[i]->ai_clist)) {
 182                                 mdb_tool_axinfo[i]->ai_clist = ic->head;
 183                                 free(ic);
 184                         }
 185                         while ((ice = mdb_tool_axinfo[i]->ai_flist)) {
 186                                 mdb_tool_axinfo[i]->ai_flist = ice->next;
 187                                 free(ice);
 188                         }
 189                 }
 190         }
 191 #endif
 192
 193         if( idcursor ) {
 194                 mdb_cursor_close( idcursor );
 195                 idcursor = NULL;
 196         }
 197         if( cursor ) {
 198                 mdb_cursor_close( cursor );
 199                 cursor = NULL;
 200         }
 201         if( mdb_tool_txn ) {
 202                 int rc;
 203                 if (( rc = mdb_txn_commit( mdb_tool_txn ))) {
 204                         Debug( LDAP_DEBUG_ANY,
 205                                 LDAP_XSTRING(mdb_tool_entry_close) ": database %s: "
 206                                 "txn_commit failed: %s (%d)\n",
 207                                 be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
 208                         return -1;
 209                 }
 210                 mdb_tool_txn = NULL;
 211         }
 212
 213         if( nholes ) {
 214                 unsigned i;
 215                 fprintf( stderr, "Error, entries missing!\n");
 216                 for (i=0; i<nholes; i++) {
 217                         fprintf(stderr, "  entry %ld: %s\n",
 218                                 holes[i].id, holes[i].dn.bv_val);
 219                 }
 220                 nholes = 0;
 221                 return -1;
 222         }
 223
 224         return 0;
 225 }
 226
 227 ID
 228 mdb_tool_entry_first_x(
 229         BackendDB *be,
 230         struct berval *base,
 231         int scope,
 232         Filter *f )
 233 {
 234         tool_base = base;
 235         tool_scope = scope;
 236         tool_filter = f;
 237
 238         return mdb_tool_entry_next( be );
 239 }
 240
 241 ID mdb_tool_entry_next(
 242         BackendDB *be )
 243 {
 244         int rc;
 245         ID id;
 246         struct mdb_info *mdb;
 247
 248         assert( be != NULL );
 249         assert( slapMode & SLAP_TOOL_MODE );
 250
 251         mdb = (struct mdb_info *) be->be_private;
 252         assert( mdb != NULL );
 253
 254         if ( !mdb_tool_txn ) {
 255                 rc = mdb_txn_begin( mdb->mi_dbenv, NULL, MDB_RDONLY, &mdb_tool_txn );
 256                 if ( rc )
 257                         return NOID;
 258                 rc = mdb_cursor_open( mdb_tool_txn, mdb->mi_id2entry, &cursor );
 259                 if ( rc ) {
 260                         mdb_txn_abort( mdb_tool_txn );
 261                         return NOID;
 262                 }
 263         }
 264
 265 next:;
 266         rc = mdb_cursor_get( cursor, &key, &data, MDB_NEXT );
 267
 268         if( rc ) {
 269                 return NOID;
 270         }
 271
 272         previd = *(ID *)key.mv_data;
 273         id = previd;
 274
 275         if ( !data.mv_size )
 276                 goto next;
 277
 278         if ( tool_filter || tool_base ) {
 279                 static Operation op = {0};
 280                 static Opheader ohdr = {0};
 281
 282                 op.o_hdr = &ohdr;
 283                 op.o_bd = be;
 284                 op.o_tmpmemctx = NULL;
 285                 op.o_tmpmfuncs = &ch_mfuncs;
 286
 287                 if ( tool_next_entry ) {
 288                         mdb_entry_release( &op, tool_next_entry, 0 );
 289                         tool_next_entry = NULL;
 290                 }
 291
 292                 rc = mdb_tool_entry_get_int( be, id, &tool_next_entry );
 293                 if ( rc == LDAP_NO_SUCH_OBJECT ) {
 294                         goto next;
 295                 }
 296
 297                 assert( tool_next_entry != NULL );
 298
 299                 if ( tool_filter && test_filter( NULL, tool_next_entry, tool_filter ) != LDAP_COMPARE_TRUE )
 300                 {
 301                         mdb_entry_release( &op, tool_next_entry, 0 );
 302                         tool_next_entry = NULL;
 303                         goto next;
 304                 }
 305         }
 306
 307         return id;
 308 }
 309
 310 ID mdb_tool_dn2id_get(
 311         Backend *be,
 312         struct berval *dn
 313 )
 314 {
 315         struct mdb_info *mdb;
 316         Operation op = {0};
 317         Opheader ohdr = {0};
 318         ID id;
 319         int rc;
 320
 321         if ( BER_BVISEMPTY(dn) )
 322                 return 0;
 323
 324         mdb = (struct mdb_info *) be->be_private;
 325
 326         if ( !mdb_tool_txn ) {
 327                 rc = mdb_txn_begin( mdb->mi_dbenv, NULL, (slapMode & SLAP_TOOL_READONLY) != 0 ?
 328                         MDB_RDONLY : 0, &mdb_tool_txn );
 329                 if ( rc )
 330                         return NOID;
 331         }
 332
 333         op.o_hdr = &ohdr;
 334         op.o_bd = be;
 335         op.o_tmpmemctx = NULL;
 336         op.o_tmpmfuncs = &ch_mfuncs;
 337
 338         rc = mdb_dn2id( &op, mdb_tool_txn, NULL, dn, &id, NULL, NULL, NULL );
 339         if ( rc == MDB_NOTFOUND )
 340                 return NOID;
 341
 342         return id;
 343 }
 344
 345 static int
 346 mdb_tool_entry_get_int( BackendDB *be, ID id, Entry **ep )
 347 {
 348         Operation op = {0};
 349         Opheader ohdr = {0};
 350
 351         Entry *e = NULL;
 352         struct berval dn = BER_BVNULL, ndn = BER_BVNULL;
 353         int rc;
 354
 355         assert( be != NULL );
 356         assert( slapMode & SLAP_TOOL_MODE );
 357
 358         if ( ( tool_filter || tool_base ) && id == previd && tool_next_entry != NULL ) {
 359                 *ep = tool_next_entry;
 360                 tool_next_entry = NULL;
 361                 return LDAP_SUCCESS;
 362         }
 363
 364         if ( id != previd ) {
 365                 key.mv_size = sizeof(ID);
 366                 key.mv_data = &id;
 367                 rc = mdb_cursor_get( cursor, &key, &data, MDB_SET );
 368                 if ( rc ) {
 369                         rc = LDAP_OTHER;
 370                         goto done;
 371                 }
 372         }
 373         if ( !data.mv_size ) {
 374                 rc = LDAP_NO_SUCH_OBJECT;
 375                 goto done;
 376         }
 377
 378         op.o_hdr = &ohdr;
 379         op.o_bd = be;
 380         op.o_tmpmemctx = NULL;
 381         op.o_tmpmfuncs = &ch_mfuncs;
 382         if ( slapMode & SLAP_TOOL_READONLY ) {
 383                 rc = mdb_id2name( &op, mdb_tool_txn, &idcursor, id, &dn, &ndn );
 384                 if ( rc  ) {
 385                         rc = LDAP_OTHER;
 386                         if ( e ) {
 387                                 mdb_entry_return( &op, e );
 388                                 e = NULL;
 389                         }
 390                         goto done;
 391                 }
 392                 if ( tool_base != NULL ) {
 393                         if ( !dnIsSuffixScope( &ndn, tool_base, tool_scope ) ) {
 394                                 ch_free( dn.bv_val );
 395                                 ch_free( ndn.bv_val );
 396                                 rc = LDAP_NO_SUCH_OBJECT;
 397                                 goto done;
 398                         }
 399                 }
 400         }
 401         rc = mdb_entry_decode( &op, mdb_tool_txn, &data, &e );
 402         e->e_id = id;
 403         if ( !BER_BVISNULL( &dn )) {
 404                 e->e_name = dn;
 405                 e->e_nname = ndn;
 406         } else {
 407                 e->e_name.bv_val = NULL;
 408                 e->e_nname.bv_val = NULL;
 409         }
 410
 411 done:
 412         if ( e != NULL ) {
 413                 *ep = e;
 414         }
 415
 416         return rc;
 417 }
 418
 419 Entry*
 420 mdb_tool_entry_get( BackendDB *be, ID id )
 421 {
 422         Entry *e = NULL;
 423         int rc;
 424
 425         if ( !mdb_tool_txn ) {
 426                 struct mdb_info *mdb = (struct mdb_info *) be->be_private;
 427                 rc = mdb_txn_begin( mdb->mi_dbenv, NULL,
 428                         (slapMode & SLAP_TOOL_READONLY) ? MDB_RDONLY : 0, &mdb_tool_txn );
 429                 if ( rc )
 430                         return NULL;
 431         }
 432         if ( !cursor ) {
 433                 struct mdb_info *mdb = (struct mdb_info *) be->be_private;
 434                 rc = mdb_cursor_open( mdb_tool_txn, mdb->mi_id2entry, &cursor );
 435                 if ( rc ) {
 436                         mdb_txn_abort( mdb_tool_txn );
 437                         mdb_tool_txn = NULL;
 438                         return NULL;
 439                 }
 440         }
 441         (void)mdb_tool_entry_get_int( be, id, &e );
 442         return e;
 443 }
 444
 445 static int mdb_tool_next_id(
 446         Operation *op,
 447         MDB_txn *tid,
 448         Entry *e,
 449         struct berval *text,
 450         int hole )
 451 {
 452         struct mdb_info *mdb = (struct mdb_info *) op->o_bd->be_private;
 453         struct berval dn = e->e_name;
 454         struct berval ndn = e->e_nname;
 455         struct berval pdn, npdn, nmatched;
 456         ID id, pid = 0;
 457         int rc;
 458
 459         if (ndn.bv_len == 0) {
 460                 e->e_id = 0;
 461                 return 0;
 462         }
 463
 464         rc = mdb_dn2id( op, tid, mcp, &ndn, &id, NULL, NULL, &nmatched );
 465         if ( rc == MDB_NOTFOUND ) {
 466                 if ( !be_issuffix( op->o_bd, &ndn ) ) {
 467                         ID eid = e->e_id;
 468                         dnParent( &ndn, &npdn );
 469                         if ( nmatched.bv_len != npdn.bv_len ) {
 470                                 dnParent( &dn, &pdn );
 471                                 e->e_name = pdn;
 472                                 e->e_nname = npdn;
 473                                 rc = mdb_tool_next_id( op, tid, e, text, 1 );
 474                                 e->e_name = dn;
 475                                 e->e_nname = ndn;
 476                                 if ( rc ) {
 477                                         return rc;
 478                                 }
 479                                 /* If parent didn't exist, it was created just now
 480                                  * and its ID is now in e->e_id. Make sure the current
 481                                  * entry gets added under the new parent ID.
 482                                  */
 483                                 if ( eid != e->e_id ) {
 484                                         pid = e->e_id;
 485                                 }
 486                         } else {
 487                                 pid = id;
 488                         }
 489                 }
 490                 rc = mdb_next_id( op->o_bd, idcursor, &e->e_id );
 491                 if ( rc ) {
 492                         snprintf( text->bv_val, text->bv_len,
 493                                 "next_id failed: %s (%d)",
 494                                 mdb_strerror(rc), rc );
 495                 Debug( LDAP_DEBUG_ANY,
 496                         "=> mdb_tool_next_id: %s\n", text->bv_val, 0, 0 );
 497                         return rc;
 498                 }
 499                 rc = mdb_dn2id_add( op, mcp, mcd, pid, 1, 1, e );
 500                 if ( rc ) {
 501                         snprintf( text->bv_val, text->bv_len,
 502                                 "dn2id_add failed: %s (%d)",
 503                                 mdb_strerror(rc), rc );
 504                         Debug( LDAP_DEBUG_ANY,
 505                                 "=> mdb_tool_next_id: %s\n", text->bv_val, 0, 0 );
 506                 } else if ( hole ) {
 507                         MDB_val key, data;
 508                         if ( nholes == nhmax - 1 ) {
 509                                 if ( holes == hbuf ) {
 510                                         holes = ch_malloc( nhmax * sizeof(dn_id) * 2 );
 511                                         AC_MEMCPY( holes, hbuf, sizeof(hbuf) );
 512                                 } else {
 513                                         holes = ch_realloc( holes, nhmax * sizeof(dn_id) * 2 );
 514                                 }
 515                                 nhmax *= 2;
 516                         }
 517                         ber_dupbv( &holes[nholes].dn, &ndn );
 518                         holes[nholes++].id = e->e_id;
 519                         key.mv_size = sizeof(ID);
 520                         key.mv_data = &e->e_id;
 521                         data.mv_size = 0;
 522                         data.mv_data = NULL;
 523                         rc = mdb_cursor_put( idcursor, &key, &data, MDB_NOOVERWRITE );
 524                         if ( rc == MDB_KEYEXIST )
 525                                 rc = 0;
 526                         if ( rc ) {
 527                                 snprintf( text->bv_val, text->bv_len,
 528                                         "dummy id2entry add failed: %s (%d)",
 529                                         mdb_strerror(rc), rc );
 530                                 Debug( LDAP_DEBUG_ANY,
 531                                         "=> mdb_tool_next_id: %s\n", text->bv_val, 0, 0 );
 532                         }
 533                 }
 534         } else if ( !hole ) {
 535                 unsigned i, j;
 536
 537                 e->e_id = id;
 538
 539                 for ( i=0; i<nholes; i++) {
 540                         if ( holes[i].id == e->e_id ) {
 541                                 free(holes[i].dn.bv_val);
 542                                 for (j=i;j<nholes;j++) holes[j] = holes[j+1];
 543                                 holes[j].id = 0;
 544                                 nholes--;
 545                                 break;
 546                         } else if ( holes[i].id > e->e_id ) {
 547                                 break;
 548                         }
 549                 }
 550         }
 551         return rc;
 552 }
 553
 554 static int
 555 mdb_tool_index_add(
 556         Operation *op,
 557         MDB_txn *txn,
 558         Entry *e )
 559 {
 560         struct mdb_info *mdb = (struct mdb_info *) op->o_bd->be_private;
 561
 562         if ( !mdb->mi_nattrs )
 563                 return 0;
 564
 565         if ( mdb_tool_threads > 1 ) {
 566                 IndexRec *ir;
 567                 int i, rc;
 568                 Attribute *a;
 569
 570                 ir = mdb_tool_index_rec;
 571                 for (i=0; i<mdb->mi_nattrs; i++)
 572                         ir[i].ir_attrs = NULL;
 573
 574                 for ( a = e->e_attrs; a != NULL; a = a->a_next ) {
 575                         rc = mdb_index_recset( mdb, a, a->a_desc->ad_type,
 576                                 &a->a_desc->ad_tags, ir );
 577                         if ( rc )
 578                                 return rc;
 579                 }
 580                 for (i=0; i<mdb->mi_nattrs; i++) {
 581                         if ( !ir[i].ir_ai )
 582                                 break;
 583                         rc = mdb_cursor_open( txn, ir[i].ir_ai->ai_dbi,
 584                                  &ir[i].ir_ai->ai_cursor );
 585                         if ( rc )
 586                                 return rc;
 587                 }
 588                 mdb_tool_ix_id = e->e_id;
 589                 mdb_tool_ix_txn = txn;
 590                 ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex );
 591                 /* Wait for all threads to be ready */
 592                 while ( mdb_tool_index_tcount ) {
 593                         ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main,
 594                                 &mdb_tool_index_mutex );
 595                 }
 596
 597                 for ( i=1; i<mdb_tool_threads; i++ )
 598                         mdb_tool_index_rec[i].ir_i = LDAP_BUSY;
 599                 mdb_tool_index_tcount = mdb_tool_threads - 1;
 600                 ldap_pvt_thread_cond_broadcast( &mdb_tool_index_cond_work );
 601                 ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex );
 602
 603                 return mdb_index_recrun( op, txn, mdb, ir, e->e_id, 0 );
 604         } else
 605         {
 606                 return mdb_index_entry_add( op, txn, e );
 607         }
 608 }
 609
 610 static int
 611 mdb_tool_index_finish()
 612 {
 613         int i, rc;
 614         ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex );
 615         for ( i=1; i<mdb_tool_threads; i++ ) {
 616                 if ( mdb_tool_index_rec[i].ir_i == LDAP_BUSY ) {
 617                         ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main,
 618                                 &mdb_tool_index_mutex );
 619                         i--;
 620                         continue;
 621                 }
 622                 if ( mdb_tool_index_rec[i].ir_i ) {
 623                         rc = mdb_tool_index_rec[i].ir_i;
 624                         break;
 625                 }
 626         }
 627         ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex );
 628         return rc;
 629 }
 630
 631 ID mdb_tool_entry_put(
 632         BackendDB *be,
 633         Entry *e,
 634         struct berval *text )
 635 {
 636         int rc;
 637         struct mdb_info *mdb;
 638         Operation op = {0};
 639         Opheader ohdr = {0};
 640
 641         assert( be != NULL );
 642         assert( slapMode & SLAP_TOOL_MODE );
 643
 644         assert( text != NULL );
 645         assert( text->bv_val != NULL );
 646         assert( text->bv_val[0] == '\0' );      /* overconservative? */
 647
 648         Debug( LDAP_DEBUG_TRACE, "=> " LDAP_XSTRING(mdb_tool_entry_put)
 649                 "( %ld, \"%s\" )\n", (long) e->e_id, e->e_dn, 0 );
 650
 651         mdb = (struct mdb_info *) be->be_private;
 652
 653         if ( !mdb_tool_txn ) {
 654                 rc = mdb_txn_begin( mdb->mi_dbenv, NULL, 0, &mdb_tool_txn );
 655                 if( rc != 0 ) {
 656                         snprintf( text->bv_val, text->bv_len,
 657                                 "txn_begin failed: %s (%d)",
 658                                 mdb_strerror(rc), rc );
 659                         Debug( LDAP_DEBUG_ANY,
 660                                 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
 661                                  text->bv_val, 0, 0 );
 662                         return NOID;
 663                 }
 664                 rc = mdb_cursor_open( mdb_tool_txn, mdb->mi_id2entry, &idcursor );
 665                 if( rc != 0 ) {
 666                         snprintf( text->bv_val, text->bv_len,
 667                                 "cursor_open failed: %s (%d)",
 668                                 mdb_strerror(rc), rc );
 669                         Debug( LDAP_DEBUG_ANY,
 670                                 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
 671                                  text->bv_val, 0, 0 );
 672                         return NOID;
 673                 }
 674                 if ( !mdb->mi_nextid ) {
 675                         ID dummy;
 676                         mdb_next_id( be, idcursor, &dummy );
 677                 }
 678                 rc = mdb_cursor_open( mdb_tool_txn, mdb->mi_dn2id, &mcp );
 679                 if( rc != 0 ) {
 680                         snprintf( text->bv_val, text->bv_len,
 681                                 "cursor_open failed: %s (%d)",
 682                                 mdb_strerror(rc), rc );
 683                         Debug( LDAP_DEBUG_ANY,
 684                                 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
 685                                  text->bv_val, 0, 0 );
 686                         return NOID;
 687                 }
 688                 rc = mdb_cursor_open( mdb_tool_txn, mdb->mi_dn2id, &mcd );
 689                 if( rc != 0 ) {
 690                         snprintf( text->bv_val, text->bv_len,
 691                                 "cursor_open failed: %s (%d)",
 692                                 mdb_strerror(rc), rc );
 693                         Debug( LDAP_DEBUG_ANY,
 694                                 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
 695                                  text->bv_val, 0, 0 );
 696                         return NOID;
 697                 }
 698         }
 699
 700         op.o_hdr = &ohdr;
 701         op.o_bd = be;
 702         op.o_tmpmemctx = NULL;
 703         op.o_tmpmfuncs = &ch_mfuncs;
 704
 705         /* add dn2id indices */
 706         rc = mdb_tool_next_id( &op, mdb_tool_txn, e, text, 0 );
 707         if( rc != 0 ) {
 708                 goto done;
 709         }
 710
 711         if ( mdb_tool_threads > 1 ) {
 712                 LDAP_SLIST_INSERT_HEAD( &op.o_extra, &mdb_tool_axinfo[0]->ai_oe, oe_next );
 713         }
 714         rc = mdb_tool_index_add( &op, mdb_tool_txn, e );
 715         if( rc != 0 ) {
 716                 snprintf( text->bv_val, text->bv_len,
 717                                 "index_entry_add failed: err=%d", rc );
 718                 Debug( LDAP_DEBUG_ANY,
 719                         "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
 720                         text->bv_val, 0, 0 );
 721                 goto done;
 722         }
 723
 724
 725         /* id2entry index */
 726         rc = mdb_id2entry_add( &op, mdb_tool_txn, idcursor, e );
 727         if( rc != 0 ) {
 728                 snprintf( text->bv_val, text->bv_len,
 729                                 "id2entry_add failed: err=%d", rc );
 730                 Debug( LDAP_DEBUG_ANY,
 731                         "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
 732                         text->bv_val, 0, 0 );
 733                 goto done;
 734         }
 735
 736         if( mdb->mi_nattrs && mdb_tool_threads > 1 )
 737                 rc = mdb_tool_index_finish();
 738
 739 done:
 740         if( rc == 0 ) {
 741                 mdb_writes++;
 742                 if ( mdb_writes >= mdb_writes_per_commit ) {
 743                         unsigned i;
 744                         MDB_TOOL_IDL_FLUSH( be, mdb_tool_txn );
 745                         rc = mdb_txn_commit( mdb_tool_txn );
 746                         for ( i=0; i<mdb->mi_nattrs; i++ )
 747                                 mdb->mi_attrs[i]->ai_cursor = NULL;
 748                         mdb_writes = 0;
 749                         mdb_tool_txn = NULL;
 750                         idcursor = NULL;
 751                         if( rc != 0 ) {
 752                                 snprintf( text->bv_val, text->bv_len,
 753                                                 "txn_commit failed: %s (%d)",
 754                                                 mdb_strerror(rc), rc );
 755                                 Debug( LDAP_DEBUG_ANY,
 756                                         "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
 757                                         text->bv_val, 0, 0 );
 758                                 e->e_id = NOID;
 759                         }
 760                 }
 761
 762         } else {
 763                 unsigned i;
 764                 mdb_txn_abort( mdb_tool_txn );
 765                 mdb_tool_txn = NULL;
 766                 idcursor = NULL;
 767                 for ( i=0; i<mdb->mi_nattrs; i++ )
 768                         mdb->mi_attrs[i]->ai_cursor = NULL;
 769                 mdb_writes = 0;
 770                 snprintf( text->bv_val, text->bv_len,
 771                         "txn_aborted! %s (%d)",
 772                         rc == LDAP_OTHER ? "Internal error" :
 773                         mdb_strerror(rc), rc );
 774                 Debug( LDAP_DEBUG_ANY,
 775                         "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
 776                         text->bv_val, 0, 0 );
 777                 e->e_id = NOID;
 778         }
 779
 780         return e->e_id;
 781 }
 782
 783 static int mdb_dn2id_upgrade( BackendDB *be );
 784
 785 int mdb_tool_entry_reindex(
 786         BackendDB *be,
 787         ID id,
 788         AttributeDescription **adv )
 789 {
 790         struct mdb_info *mi = (struct mdb_info *) be->be_private;
 791         int rc;
 792         Entry *e;
 793         Operation op = {0};
 794         Opheader ohdr = {0};
 795
 796         Debug( LDAP_DEBUG_ARGS,
 797                 "=> " LDAP_XSTRING(mdb_tool_entry_reindex) "( %ld )\n",
 798                 (long) id, 0, 0 );
 799         assert( tool_base == NULL );
 800         assert( tool_filter == NULL );
 801
 802         /* Special: do a dn2id upgrade */
 803         if ( adv && adv[0] == slap_schema.si_ad_entryDN ) {
 804                 /* short-circuit tool_entry_next() */
 805                 mdb_cursor_get( cursor, &key, &data, MDB_LAST );
 806                 return mdb_dn2id_upgrade( be );
 807         }
 808
 809         /* No indexes configured, nothing to do. Could return an
 810          * error here to shortcut things.
 811          */
 812         if (!mi->mi_attrs) {
 813                 return 0;
 814         }
 815
 816         /* Check for explicit list of attrs to index */
 817         if ( adv ) {
 818                 int i, j, n;
 819
 820                 if ( mi->mi_attrs[0]->ai_desc != adv[0] ) {
 821                         /* count */
 822                         for ( n = 0; adv[n]; n++ ) ;
 823
 824                         /* insertion sort */
 825                         for ( i = 0; i < n; i++ ) {
 826                                 AttributeDescription *ad = adv[i];
 827                                 for ( j = i-1; j>=0; j--) {
 828                                         if ( SLAP_PTRCMP( adv[j], ad ) <= 0 ) break;
 829                                         adv[j+1] = adv[j];
 830                                 }
 831                                 adv[j+1] = ad;
 832                         }
 833                 }
 834
 835                 for ( i = 0; adv[i]; i++ ) {
 836                         if ( mi->mi_attrs[i]->ai_desc != adv[i] ) {
 837                                 for ( j = i+1; j < mi->mi_nattrs; j++ ) {
 838                                         if ( mi->mi_attrs[j]->ai_desc == adv[i] ) {
 839                                                 AttrInfo *ai = mi->mi_attrs[i];
 840                                                 mi->mi_attrs[i] = mi->mi_attrs[j];
 841                                                 mi->mi_attrs[j] = ai;
 842                                                 break;
 843                                         }
 844                                 }
 845                                 if ( j == mi->mi_nattrs ) {
 846                                         Debug( LDAP_DEBUG_ANY,
 847                                                 LDAP_XSTRING(mdb_tool_entry_reindex)
 848                                                 ": no index configured for %s\n",
 849                                                 adv[i]->ad_cname.bv_val, 0, 0 );
 850                                         return -1;
 851                                 }
 852                         }
 853                 }
 854                 mi->mi_nattrs = i;
 855         }
 856
 857         e = mdb_tool_entry_get( be, id );
 858
 859         if( e == NULL ) {
 860                 Debug( LDAP_DEBUG_ANY,
 861                         LDAP_XSTRING(mdb_tool_entry_reindex)
 862                         ": could not locate id=%ld\n",
 863                         (long) id, 0, 0 );
 864                 return -1;
 865         }
 866
 867         if ( !txi ) {
 868                 rc = mdb_txn_begin( mi->mi_dbenv, NULL, 0, &txi );
 869                 if( rc != 0 ) {
 870                         Debug( LDAP_DEBUG_ANY,
 871                                 "=> " LDAP_XSTRING(mdb_tool_entry_reindex) ": "
 872                                 "txn_begin failed: %s (%d)\n",
 873                                 mdb_strerror(rc), rc, 0 );
 874                         goto done;
 875                 }
 876         }
 877
 878         if ( slapMode & SLAP_TRUNCATE_MODE ) {
 879                 int i;
 880                 for ( i=0; i < mi->mi_nattrs; i++ ) {
 881                         rc = mdb_drop( txi, mi->mi_attrs[i]->ai_dbi, 0 );
 882                         if ( rc ) {
 883                                 Debug( LDAP_DEBUG_ANY,
 884                                         LDAP_XSTRING(mdb_tool_entry_reindex)
 885                                         ": (Truncate) mdb_drop(%s) failed: %s (%d)\n",
 886                                         mi->mi_attrs[i]->ai_desc->ad_type->sat_cname.bv_val,
 887                                         mdb_strerror(rc), rc );
 888                                 return -1;
 889                         }
 890                 }
 891                 slapMode ^= SLAP_TRUNCATE_MODE;
 892         }
 893
 894         /*
 895          * just (re)add them for now
 896          * Use truncate mode to empty/reset index databases
 897          */
 898
 899         Debug( LDAP_DEBUG_TRACE,
 900                 "=> " LDAP_XSTRING(mdb_tool_entry_reindex) "( %ld )\n",
 901                 (long) id, 0, 0 );
 902
 903         op.o_hdr = &ohdr;
 904         op.o_bd = be;
 905         op.o_tmpmemctx = NULL;
 906         op.o_tmpmfuncs = &ch_mfuncs;
 907
 908         rc = mdb_tool_index_add( &op, txi, e );
 909
 910 done:
 911         if( rc == 0 ) {
 912                 mdb_writes++;
 913                 if ( mdb_writes >= mdb_writes_per_commit ) {
 914                         MDB_val key;
 915                         unsigned i;
 916                         MDB_TOOL_IDL_FLUSH( be, txi );
 917                         rc = mdb_txn_commit( txi );
 918                         mdb_writes = 0;
 919                         for ( i=0; i<mi->mi_nattrs; i++ )
 920                                 mi->mi_attrs[i]->ai_cursor = NULL;
 921                         if( rc != 0 ) {
 922                                 Debug( LDAP_DEBUG_ANY,
 923                                         "=> " LDAP_XSTRING(mdb_tool_entry_reindex)
 924                                         ": txn_commit failed: %s (%d)\n",
 925                                         mdb_strerror(rc), rc, 0 );
 926                                 e->e_id = NOID;
 927                         }
 928                         mdb_cursor_close( cursor );
 929                         txi = NULL;
 930                         /* Must close the read txn to allow old pages to be reclaimed. */
 931                         mdb_txn_abort( mdb_tool_txn );
 932                         /* and then reopen it so that tool_entry_next still works. */
 933                         mdb_txn_begin( mi->mi_dbenv, NULL, MDB_RDONLY, &mdb_tool_txn );
 934                         mdb_cursor_open( mdb_tool_txn, mi->mi_id2entry, &cursor );
 935                         key.mv_data = &id;
 936                         key.mv_size = sizeof(ID);
 937                         mdb_cursor_get( cursor, &key, NULL, MDB_SET );
 938                 }
 939
 940         } else {
 941                 unsigned i;
 942                 mdb_writes = 0;
 943                 mdb_cursor_close( cursor );
 944                 cursor = NULL;
 945                 mdb_txn_abort( txi );
 946                 for ( i=0; i<mi->mi_nattrs; i++ )
 947                         mi->mi_attrs[i]->ai_cursor = NULL;
 948                 Debug( LDAP_DEBUG_ANY,
 949                         "=> " LDAP_XSTRING(mdb_tool_entry_reindex)
 950                         ": txn_aborted! err=%d\n",
 951                         rc, 0, 0 );
 952                 e->e_id = NOID;
 953                 txi = NULL;
 954         }
 955         mdb_entry_release( &op, e, 0 );
 956
 957         return rc;
 958 }
 959
 960 ID mdb_tool_entry_modify(
 961         BackendDB *be,
 962         Entry *e,
 963         struct berval *text )
 964 {
 965         int rc;
 966         struct mdb_info *mdb;
 967         Operation op = {0};
 968         Opheader ohdr = {0};
 969
 970         assert( be != NULL );
 971         assert( slapMode & SLAP_TOOL_MODE );
 972
 973         assert( text != NULL );
 974         assert( text->bv_val != NULL );
 975         assert( text->bv_val[0] == '\0' );      /* overconservative? */
 976
 977         assert ( e->e_id != NOID );
 978
 979         Debug( LDAP_DEBUG_TRACE,
 980                 "=> " LDAP_XSTRING(mdb_tool_entry_modify) "( %ld, \"%s\" )\n",
 981                 (long) e->e_id, e->e_dn, 0 );
 982
 983         mdb = (struct mdb_info *) be->be_private;
 984
 985         if( cursor ) {
 986                 mdb_cursor_close( cursor );
 987                 cursor = NULL;
 988         }
 989         if ( !mdb_tool_txn ) {
 990                 rc = mdb_txn_begin( mdb->mi_dbenv, NULL, 0, &mdb_tool_txn );
 991                 if( rc != 0 ) {
 992                         snprintf( text->bv_val, text->bv_len,
 993                                 "txn_begin failed: %s (%d)",
 994                                 mdb_strerror(rc), rc );
 995                         Debug( LDAP_DEBUG_ANY,
 996                                 "=> " LDAP_XSTRING(mdb_tool_entry_modify) ": %s\n",
 997                                  text->bv_val, 0, 0 );
 998                         return NOID;
 999                 }
1000         }
1001
1002         op.o_hdr = &ohdr;
1003         op.o_bd = be;
1004         op.o_tmpmemctx = NULL;
1005         op.o_tmpmfuncs = &ch_mfuncs;
1006
1007         /* id2entry index */
1008         rc = mdb_id2entry_update( &op, mdb_tool_txn, NULL, e );
1009         if( rc != 0 ) {
1010                 snprintf( text->bv_val, text->bv_len,
1011                                 "id2entry_update failed: err=%d", rc );
1012                 Debug( LDAP_DEBUG_ANY,
1013                         "=> " LDAP_XSTRING(mdb_tool_entry_modify) ": %s\n",
1014                         text->bv_val, 0, 0 );
1015                 goto done;
1016         }
1017
1018 done:
1019         if( rc == 0 ) {
1020                 rc = mdb_txn_commit( mdb_tool_txn );
1021                 if( rc != 0 ) {
1022                         snprintf( text->bv_val, text->bv_len,
1023                                         "txn_commit failed: %s (%d)",
1024                                         mdb_strerror(rc), rc );
1025                         Debug( LDAP_DEBUG_ANY,
1026                                 "=> " LDAP_XSTRING(mdb_tool_entry_modify) ": "
1027                                 "%s\n", text->bv_val, 0, 0 );
1028                         e->e_id = NOID;
1029                 }
1030
1031         } else {
1032                 mdb_txn_abort( mdb_tool_txn );
1033                 snprintf( text->bv_val, text->bv_len,
1034                         "txn_aborted! %s (%d)",
1035                         mdb_strerror(rc), rc );
1036                 Debug( LDAP_DEBUG_ANY,
1037                         "=> " LDAP_XSTRING(mdb_tool_entry_modify) ": %s\n",
1038                         text->bv_val, 0, 0 );
1039                 e->e_id = NOID;
1040         }
1041         mdb_tool_txn = NULL;
1042         idcursor = NULL;
1043
1044         return e->e_id;
1045 }
1046
1047 static void *
1048 mdb_tool_index_task( void *ctx, void *ptr )
1049 {
1050         int base = *(int *)ptr;
1051         Operation op = {0};
1052         Opheader ohdr = {0};
1053         AttrIxInfo ai = {0}, *aio;
1054
1055         free( ptr );
1056         op.o_hdr = &ohdr;
1057         op.o_bd = mdb_tool_ix_be;
1058         op.o_tmpmemctx = NULL;
1059         op.o_tmpmfuncs = &ch_mfuncs;
1060         aio = mdb_tool_axinfo[base];
1061         mdb_tool_axinfo[base] = &ai;
1062         LDAP_SLIST_INSERT_HEAD( &op.o_extra, &ai.ai_oe, oe_next );
1063         while ( 1 ) {
1064                 ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex );
1065                 mdb_tool_index_tcount--;
1066                 if ( !mdb_tool_index_tcount )
1067                         ldap_pvt_thread_cond_signal( &mdb_tool_index_cond_main );
1068                 ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_work,
1069                         &mdb_tool_index_mutex );
1070                 if ( slapd_shutdown ) {
1071                         mdb_tool_index_tcount--;
1072                         if ( !mdb_tool_index_tcount )
1073                                 ldap_pvt_thread_cond_signal( &mdb_tool_index_cond_main );
1074                         *aio = ai;
1075                         mdb_tool_axinfo[base] = aio;
1076                         ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex );
1077                         break;
1078                 }
1079                 ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex );
1080                 mdb_tool_index_rec[base].ir_i = mdb_index_recrun( &op,
1081                         mdb_tool_ix_txn,
1082                         mdb_tool_info, mdb_tool_index_rec, mdb_tool_ix_id, base );
1083         }
1084
1085         return NULL;
1086 }
1087
1088 #ifdef MDB_TOOL_IDL_CACHING
1089 static int
1090 mdb_tool_idl_cmp( const void *v1, const void *v2 )
1091 {
1092         const mdb_tool_idl_cache *c1 = v1, *c2 = v2;
1093         int rc;
1094
1095         if (( rc = c1->kstr.bv_len - c2->kstr.bv_len )) return rc;
1096         return memcmp( c1->kstr.bv_val, c2->kstr.bv_val, c1->kstr.bv_len );
1097 }
1098
1099 static int
1100 mdb_tool_idl_flush_one( MDB_cursor *mc, AttrIxInfo *ai, mdb_tool_idl_cache *ic )
1101 {
1102         mdb_tool_idl_cache_entry *ice;
1103         MDB_val key, data[2];
1104         int i, rc;
1105         ID id, nid;
1106
1107         /* Freshly allocated, ignore it */
1108         if ( !ic->head && ic->count <= MDB_IDL_DB_SIZE ) {
1109                 return 0;
1110         }
1111
1112         key.mv_data = ic->kstr.bv_val;
1113         key.mv_size = ic->kstr.bv_len;
1114
1115         if ( ic->count > MDB_IDL_DB_SIZE ) {
1116                 while ( ic->flags & WAS_FOUND ) {
1117                         rc = mdb_cursor_get( mc, &key, data, MDB_SET );
1118                         if ( rc ) {
1119                                 /* FIXME: find out why this happens */
1120                                 ic->flags = 0;
1121                                 break;
1122                         }
1123                         if ( ic->flags & WAS_RANGE ) {
1124                                 /* Skip lo */
1125                                 rc = mdb_cursor_get( mc, &key, data, MDB_NEXT_DUP );
1126
1127                                 /* Get hi */
1128                                 rc = mdb_cursor_get( mc, &key, data, MDB_NEXT_DUP );
1129
1130                                 /* Store range hi */
1131                                 data[0].mv_data = &ic->last;
1132                                 rc = mdb_cursor_put( mc, &key, data, MDB_CURRENT );
1133                         } else {
1134                                 /* Delete old data, replace with range */
1135                                 ic->first = *(ID *)data[0].mv_data;
1136                                 mdb_cursor_del( mc, MDB_NODUPDATA );
1137                         }
1138                         break;
1139                 }
1140                 if ( !(ic->flags & WAS_RANGE)) {
1141                         /* range, didn't exist before */
1142                         nid = 0;
1143                         data[0].mv_size = sizeof(ID);
1144                         data[0].mv_data = &nid;
1145                         rc = mdb_cursor_put( mc, &key, data, 0 );
1146                         if ( rc == 0 ) {
1147                                 data[0].mv_data = &ic->first;
1148                                 rc = mdb_cursor_put( mc, &key, data, 0 );
1149                                 if ( rc == 0 ) {
1150                                         data[0].mv_data = &ic->last;
1151                                         rc = mdb_cursor_put( mc, &key, data, 0 );
1152                                 }
1153                         }
1154                         if ( rc ) {
1155                                 rc = -1;
1156                         }
1157                 }
1158         } else {
1159                 /* Normal write */
1160                 int n;
1161
1162                 data[0].mv_size = sizeof(ID);
1163                 rc = 0;
1164                 for ( ice = ic->head, n=0; ice; ice = ice->next, n++ ) {
1165                         int end;
1166                         if ( ice->next ) {
1167                                 end = IDBLOCK;
1168                         } else {
1169                                 end = (ic->count-ic->offset) & (IDBLOCK-1);
1170                                 if ( !end )
1171                                         end = IDBLOCK;
1172                         }
1173                         data[1].mv_size = end;
1174                         data[0].mv_data = ice->ids;
1175                         rc = mdb_cursor_put( mc, &key, data, MDB_APPENDDUP|MDB_MULTIPLE );
1176                         if ( rc ) {
1177                                 rc = -1;
1178                                 break;
1179                         }
1180                 }
1181                 if ( ic->head ) {
1182                         ic->tail->next = ai->ai_flist;
1183                         ai->ai_flist = ic->head;
1184                 }
1185         }
1186         ic->head = ai->ai_clist;
1187         ai->ai_clist = ic;
1188         return rc;
1189 }
1190
1191 static int
1192 mdb_tool_idl_flush_db( MDB_txn *txn, AttrInfo *ai, AttrIxInfo *ax )
1193 {
1194         MDB_cursor *mc;
1195         Avlnode *root;
1196         int rc;
1197
1198         mdb_cursor_open( txn, ai->ai_dbi, &mc );
1199         root = tavl_end( ai->ai_root, TAVL_DIR_LEFT );
1200         do {
1201                 rc = mdb_tool_idl_flush_one( mc, ax, root->avl_data );
1202                 if ( rc != -1 )
1203                         rc = 0;
1204         } while ((root = tavl_next(root, TAVL_DIR_RIGHT)));
1205         mdb_cursor_close( mc );
1206
1207         return rc;
1208 }
1209
1210 static int
1211 mdb_tool_idl_flush( BackendDB *be, MDB_txn *txn )
1212 {
1213         struct mdb_info *mdb = (struct mdb_info *) be->be_private;
1214         int rc = 0;
1215         unsigned int i, dbi;
1216
1217         for ( i=0; i < mdb->mi_nattrs; i++ ) {
1218                 if ( !mdb->mi_attrs[i]->ai_root ) continue;
1219                 rc = mdb_tool_idl_flush_db( txn, mdb->mi_attrs[i], mdb_tool_axinfo[i % mdb_tool_threads] );
1220                 tavl_free(mdb->mi_attrs[i]->ai_root, NULL);
1221                 mdb->mi_attrs[i]->ai_root = NULL;
1222                 if ( rc )
1223                         break;
1224         }
1225         return rc;
1226 }
1227
1228 int mdb_tool_idl_add(
1229         BackendDB *be,
1230         MDB_cursor *mc,
1231         struct berval *keys,
1232         ID id )
1233 {
1234         MDB_dbi dbi;
1235         mdb_tool_idl_cache *ic, itmp;
1236         mdb_tool_idl_cache_entry *ice;
1237         int i, rc, lcount;
1238         AttrIxInfo *ax = (AttrIxInfo *)mc;
1239         AttrInfo *ai = (AttrInfo *)ax->ai_ai;
1240         mc = ai->ai_cursor;
1241
1242         dbi = ai->ai_dbi;
1243         for (i=0; keys[i].bv_val; i++) {
1244         itmp.kstr = keys[i];
1245         ic = tavl_find( (Avlnode *)ai->ai_root, &itmp, mdb_tool_idl_cmp );
1246
1247         /* No entry yet, create one */
1248         if ( !ic ) {
1249                 MDB_val key, data;
1250                 ID nid;
1251                 int rc;
1252
1253                 if ( ax->ai_clist ) {
1254                         ic = ax->ai_clist;
1255                         ax->ai_clist = ic->head;
1256                 } else {
1257                         ic = ch_malloc( sizeof( mdb_tool_idl_cache ) + itmp.kstr.bv_len + 4 );
1258                 }
1259                 ic->kstr.bv_len = itmp.kstr.bv_len;
1260                 ic->kstr.bv_val = (char *)(ic+1);
1261                 memcpy( ic->kstr.bv_val, itmp.kstr.bv_val, ic->kstr.bv_len );
1262                 ic->head = ic->tail = NULL;
1263                 ic->last = 0;
1264                 ic->count = 0;
1265                 ic->offset = 0;
1266                 ic->flags = 0;
1267                 tavl_insert( (Avlnode **)&ai->ai_root, ic, mdb_tool_idl_cmp,
1268                         avl_dup_error );
1269
1270                 /* load existing key count here */
1271                 key.mv_size = keys[i].bv_len;
1272                 key.mv_data = keys[i].bv_val;
1273                 rc = mdb_cursor_get( mc, &key, &data, MDB_SET );
1274                 if ( rc == 0 ) {
1275                         ic->flags |= WAS_FOUND;
1276                         nid = *(ID *)data.mv_data;
1277                         if ( nid == 0 ) {
1278                                 ic->count = MDB_IDL_DB_SIZE+1;
1279                                 ic->flags |= WAS_RANGE;
1280                         } else {
1281                                 size_t count;
1282
1283                                 mdb_cursor_count( mc, &count );
1284                                 ic->count = count;
1285                                 ic->first = nid;
1286                                 ic->offset = count & (IDBLOCK-1);
1287                         }
1288                 }
1289         }
1290         /* are we a range already? */
1291         if ( ic->count > MDB_IDL_DB_SIZE ) {
1292                 ic->last = id;
1293                 continue;
1294         /* Are we at the limit, and converting to a range? */
1295         } else if ( ic->count == MDB_IDL_DB_SIZE ) {
1296                 if ( ic->head ) {
1297                         ic->tail->next = ax->ai_flist;
1298                         ax->ai_flist = ic->head;
1299                 }
1300                 ic->head = ic->tail = NULL;
1301                 ic->last = id;
1302                 ic->count++;
1303                 continue;
1304         }
1305         /* No free block, create that too */
1306         lcount = (ic->count-ic->offset) & (IDBLOCK-1);
1307         if ( !ic->tail || lcount == 0) {
1308                 if ( ax->ai_flist ) {
1309                         ice = ax->ai_flist;
1310                         ax->ai_flist = ice->next;
1311                 } else {
1312                         ice = ch_malloc( sizeof( mdb_tool_idl_cache_entry ));
1313                 }
1314                 ice->next = NULL;
1315                 if ( !ic->head ) {
1316                         ic->head = ice;
1317                 } else {
1318                         ic->tail->next = ice;
1319                 }
1320                 ic->tail = ice;
1321                 if ( lcount )
1322                         ice->ids[lcount-1] = 0;
1323                 if ( !ic->count )
1324                         ic->first = id;
1325         }
1326         ice = ic->tail;
1327         if (!lcount || ice->ids[lcount-1] != id) {
1328                 ice->ids[lcount] = id;
1329                 ic->count++;
1330         }
1331         }
1332
1333         return 0;
1334 }
1335 #endif /* MDB_TOOL_IDL_CACHING */
1336
1337 /* Upgrade from pre 2.4.34 dn2id format */
1338
1339 #include <ac/unistd.h>
1340 #include <lutil_meter.h>
1341
1342 #define STACKSIZ        2048
1343
1344 typedef struct rec {
1345         ID id;
1346         size_t len;
1347         char rdn[512];
1348 } rec;
1349
1350 static int
1351 mdb_dn2id_upgrade( BackendDB *be ) {
1352         struct mdb_info *mi = (struct mdb_info *) be->be_private;
1353         MDB_txn *mt;
1354         MDB_cursor *mc = NULL;
1355         MDB_val key, data;
1356         char *ptr;
1357         int rc, writes=0, depth=0;
1358         int enable_meter = 0;
1359         ID id = 0, *num, count = 0;
1360         rec *stack;
1361         lutil_meter_t meter;
1362
1363         if (!(mi->mi_flags & MDB_NEED_UPGRADE)) {
1364                 Debug( LDAP_DEBUG_ANY, "database %s: No upgrade needed.\n",
1365                         be->be_suffix[0].bv_val, 0, 0 );
1366                 return 0;
1367         }
1368
1369         {
1370                 MDB_stat st;
1371
1372                 mdb_stat(mdb_cursor_txn(cursor), mi->mi_dbis[MDB_ID2ENTRY], &st);
1373                 if (!st.ms_entries) {
1374                         /* Empty DB, nothing to upgrade? */
1375                         return 0;
1376                 }
1377                 if (isatty(2))
1378                         enable_meter = !lutil_meter_open(&meter,
1379                                 &lutil_meter_text_display,
1380                                 &lutil_meter_linear_estimator,
1381                                 st.ms_entries);
1382         }
1383
1384         num = ch_malloc(STACKSIZ * (sizeof(ID) + sizeof(rec)));
1385         stack = (rec *)(num + STACKSIZ);
1386
1387         rc = mdb_txn_begin(mi->mi_dbenv, NULL, 0, &mt);
1388         if (rc) {
1389                 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_begin failed, %s (%d)\n",
1390                         mdb_strerror(rc), rc, 0 );
1391                 goto leave;
1392         }
1393         rc = mdb_cursor_open(mt, mi->mi_dbis[MDB_DN2ID], &mc);
1394         if (rc) {
1395                 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_open failed, %s (%d)\n",
1396                         mdb_strerror(rc), rc, 0 );
1397                 goto leave;
1398         }
1399
1400         key.mv_size = sizeof(ID);
1401         /* post-order depth-first update */
1402         for(;;) {
1403                 size_t dkids;
1404                 unsigned char *ptr;
1405
1406                 /* visit */
1407                 key.mv_data = &id;
1408                 stack[depth].id = id;
1409                 rc = mdb_cursor_get(mc, &key, &data, MDB_SET);
1410                 if (rc) {
1411                         Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_get failed, %s (%d)\n",
1412                                 mdb_strerror(rc), rc, 0 );
1413                         goto leave;
1414                 }
1415                 num[depth] = 1;
1416
1417                 rc = mdb_cursor_count(mc, &dkids);
1418                 if (rc) {
1419                         Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_count failed, %s (%d)\n",
1420                                 mdb_strerror(rc), rc, 0 );
1421                         goto leave;
1422                 }
1423                 if (dkids > 1) {
1424                         rc = mdb_cursor_get(mc, &key, &data, MDB_NEXT_DUP);
1425 down:
1426                         ptr = (unsigned char *)data.mv_data + data.mv_size - sizeof(ID);
1427                         memcpy(&id, ptr, sizeof(ID));
1428                         depth++;
1429                         memcpy(stack[depth].rdn, data.mv_data, data.mv_size);
1430                         stack[depth].len = data.mv_size;
1431                         continue;
1432                 }
1433
1434
1435                 /* pop: write updated count, advance to next node */
1436 pop:
1437                 /* update superior counts */
1438                 if (depth)
1439                         num[depth-1] += num[depth];
1440
1441                 key.mv_data = &id;
1442                 id = stack[depth-1].id;
1443                 data.mv_data = stack[depth].rdn;
1444                 data.mv_size = stack[depth].len;
1445                 rc = mdb_cursor_get(mc, &key, &data, MDB_GET_BOTH);
1446                 if (rc) {
1447                         Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_get(BOTH) failed, %s (%d)\n",
1448                                 mdb_strerror(rc), rc, 0 );
1449                         goto leave;
1450                 }
1451                 data.mv_data = stack[depth].rdn;
1452                 ptr = (unsigned char *)data.mv_data + data.mv_size;
1453                 memcpy(ptr, &num[depth], sizeof(ID));
1454                 data.mv_size += sizeof(ID);
1455                 rc = mdb_cursor_del(mc, 0);
1456                 if (rc) {
1457                         Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_del failed, %s (%d)\n",
1458                                 mdb_strerror(rc), rc, 0 );
1459                         goto leave;
1460                 }
1461                 rc = mdb_cursor_put(mc, &key, &data, 0);
1462                 if (rc) {
1463                         Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_put failed, %s (%d)\n",
1464                                 mdb_strerror(rc), rc, 0 );
1465                         goto leave;
1466                 }
1467                 count++;
1468 #if 1
1469                 if (enable_meter)
1470                         lutil_meter_update(&meter, count, 0);
1471 #else
1472                 {
1473                         int len;
1474                         ptr = data.mv_data;
1475                         len = (ptr[0] & 0x7f) << 8 | ptr[1];
1476                         printf("ID: %zu, %zu, %.*s\n", stack[depth].id, num[depth], len, ptr+2);
1477                 }
1478 #endif
1479                 writes++;
1480                 if (writes == 1000) {
1481                         mdb_cursor_close(mc);
1482                         rc = mdb_txn_commit(mt);
1483                         if (rc) {
1484                                 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_commit failed, %s (%d)\n",
1485                                         mdb_strerror(rc), rc, 0 );
1486                                 goto leave;
1487                         }
1488                         rc = mdb_txn_begin(mi->mi_dbenv, NULL, 0, &mt);
1489                         if (rc) {
1490                                 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_begin(2) failed, %s (%d)\n",
1491                                         mdb_strerror(rc), rc, 0 );
1492                                 goto leave;
1493                         }
1494                         rc = mdb_cursor_open(mt, mi->mi_dbis[MDB_DN2ID], &mc);
1495                         if (rc) {
1496                                 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_open(2) failed, %s (%d)\n",
1497                                         mdb_strerror(rc), rc, 0 );
1498                                 goto leave;
1499                         }
1500                         rc = mdb_cursor_get(mc, &key, &data, MDB_GET_BOTH);
1501                         if (rc) {
1502                                 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_get(2) failed, %s (%d)\n",
1503                                         mdb_strerror(rc), rc, 0 );
1504                                 goto leave;
1505                         }
1506                         writes = 0;
1507                 }
1508                 depth--;
1509
1510                 rc = mdb_cursor_get(mc, &key, &data, MDB_NEXT_DUP);
1511                 if (rc == 0)
1512                         goto down;
1513                 rc = 0;
1514                 if (depth)
1515                         goto pop;
1516                 else
1517                         break;
1518         }
1519 leave:
1520         mdb_cursor_close(mc);
1521         if (mt) {
1522                 int r2;
1523                 r2 = mdb_txn_commit(mt);
1524                 if (r2) {
1525                         Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_commit(2) failed, %s (%d)\n",
1526                                 mdb_strerror(r2), r2, 0 );
1527                         if (!rc)
1528                                 rc = r2;
1529                 }
1530         }
1531         ch_free(num);
1532         if (enable_meter) {
1533                 lutil_meter_update(&meter, count, 1);
1534                 lutil_meter_close(&meter);
1535         }
1536         return rc;
1537 }