git.sur5r.net Git - openldap/blob - servers/slapd/back-mdb/tools.c

   1 /* tools.c - tools for slap tools */
   2 /* $OpenLDAP$ */
   3 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
   4  *
   5  * Copyright 2011 The OpenLDAP Foundation.
   6  * All rights reserved.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted only as authorized by the OpenLDAP
  10  * Public License.
  11  *
  12  * A copy of this license is available in the file LICENSE in the
  13  * top-level directory of the distribution or, alternatively, at
  14  * <http://www.OpenLDAP.org/license.html>.
  15  */
  16
  17 #include "portable.h"
  18
  19 #include <stdio.h>
  20 #include <ac/string.h>
  21 #include <ac/errno.h>
  22
  23 #define AVL_INTERNAL
  24 #include "back-mdb.h"
  25 #include "idl.h"
  26
  27 #ifdef MDB_TOOL_IDL_CACHING
  28 static int mdb_tool_idl_flush( BackendDB *be, MDB_txn *txn );
  29
  30 #define IDBLOCK 1024
  31
  32 typedef struct mdb_tool_idl_cache_entry {
  33         struct mdb_tool_idl_cache_entry *next;
  34         ID ids[IDBLOCK];
  35 } mdb_tool_idl_cache_entry;
  36
  37 typedef struct mdb_tool_idl_cache {
  38         struct berval kstr;
  39         mdb_tool_idl_cache_entry *head, *tail;
  40         ID first, last;
  41         int count;
  42         short offset;
  43         short flags;
  44 } mdb_tool_idl_cache;
  45 #define WAS_FOUND       0x01
  46 #define WAS_RANGE       0x02
  47
  48 static mdb_tool_idl_cache_entry *mdb_tool_idl_free_list;
  49 static Avlnode *mdb_tool_roots[MDB_INDICES];
  50
  51 #define MDB_TOOL_IDL_FLUSH(be, txn)     mdb_tool_idl_flush(be, txn)
  52 #else
  53 #define MDB_TOOL_IDL_FLUSH(be, txn)
  54 #endif /* MDB_TOOL_IDL_CACHING */
  55
  56 static MDB_txn *txn = NULL, *txi = NULL;
  57 static MDB_cursor *cursor = NULL, *idcursor = NULL;
  58 static MDB_val key, data;
  59 static ID previd = NOID;
  60
  61 typedef struct dn_id {
  62         ID id;
  63         struct berval dn;
  64 } dn_id;
  65
  66 #define HOLE_SIZE       4096
  67 static dn_id hbuf[HOLE_SIZE], *holes = hbuf;
  68 static unsigned nhmax = HOLE_SIZE;
  69 static unsigned nholes;
  70
  71 static struct berval    *tool_base;
  72 static int              tool_scope;
  73 static Filter           *tool_filter;
  74 static Entry            *tool_next_entry;
  75
  76 #if 0
  77 static ID mdb_tool_ix_id;
  78 static Operation *mdb_tool_ix_op;
  79 static int *mdb_tool_index_threads, mdb_tool_index_tcount;
  80 static void *mdb_tool_index_rec;
  81 static struct mdb_info *mdb_tool_info;
  82 static ldap_pvt_thread_mutex_t mdb_tool_index_mutex;
  83 static ldap_pvt_thread_cond_t mdb_tool_index_cond_main;
  84 static ldap_pvt_thread_cond_t mdb_tool_index_cond_work;
  85 static void * mdb_tool_index_task( void *ctx, void *ptr );
  86 #endif
  87
  88 static int      mdb_writes, mdb_writes_per_commit;
  89
  90 static int
  91 mdb_tool_entry_get_int( BackendDB *be, ID id, Entry **ep );
  92
  93 int mdb_tool_entry_open(
  94         BackendDB *be, int mode )
  95 {
  96         /* In Quick mode, commit once per 1000 entries */
  97         mdb_writes = 0;
  98         if ( slapMode & SLAP_TOOL_QUICK )
  99                 mdb_writes_per_commit = 1000;
 100         else
 101                 mdb_writes_per_commit = 1;
 102
 103 #if 0
 104         /* Set up for threaded slapindex */
 105         if (( slapMode & (SLAP_TOOL_QUICK|SLAP_TOOL_READONLY)) == SLAP_TOOL_QUICK ) {
 106                 if ( !mdb_tool_info ) {
 107                         ldap_pvt_thread_mutex_init( &mdb_tool_index_mutex );
 108                         ldap_pvt_thread_cond_init( &mdb_tool_index_cond_main );
 109                         ldap_pvt_thread_cond_init( &mdb_tool_index_cond_work );
 110                         if ( mdb->bi_nattrs ) {
 111                                 int i;
 112                                 mdb_tool_index_threads = ch_malloc( slap_tool_thread_max * sizeof( int ));
 113                                 mdb_tool_index_rec = ch_malloc( mdb->bi_nattrs * sizeof( IndexRec ));
 114                                 mdb_tool_index_tcount = slap_tool_thread_max - 1;
 115                                 for (i=1; i<slap_tool_thread_max; i++) {
 116                                         int *ptr = ch_malloc( sizeof( int ));
 117                                         *ptr = i;
 118                                         ldap_pvt_thread_pool_submit( &connection_pool,
 119                                                 mdb_tool_index_task, ptr );
 120                                 }
 121                         }
 122                         mdb_tool_info = mdb;
 123                 }
 124         }
 125 #endif
 126
 127         return 0;
 128 }
 129
 130 int mdb_tool_entry_close(
 131         BackendDB *be )
 132 {
 133 #if 0
 134         if ( mdb_tool_info ) {
 135                 slapd_shutdown = 1;
 136                 ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex );
 137
 138                 /* There might still be some threads starting */
 139                 while ( mdb_tool_index_tcount ) {
 140                         ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main,
 141                                         &mdb_tool_index_mutex );
 142                 }
 143
 144                 mdb_tool_index_tcount = slap_tool_thread_max - 1;
 145                 ldap_pvt_thread_cond_broadcast( &mdb_tool_index_cond_work );
 146
 147                 /* Make sure all threads are stopped */
 148                 while ( mdb_tool_index_tcount ) {
 149                         ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main,
 150                                 &mdb_tool_index_mutex );
 151                 }
 152                 ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex );
 153
 154                 mdb_tool_info = NULL;
 155                 slapd_shutdown = 0;
 156                 ch_free( mdb_tool_index_threads );
 157                 ch_free( mdb_tool_index_rec );
 158                 mdb_tool_index_tcount = slap_tool_thread_max - 1;
 159         }
 160 #endif
 161
 162         if( idcursor ) {
 163                 mdb_cursor_close( idcursor );
 164                 idcursor = NULL;
 165         }
 166         if( cursor ) {
 167                 mdb_cursor_close( cursor );
 168                 cursor = NULL;
 169         }
 170         MDB_TOOL_IDL_FLUSH( be, txn );
 171         if( txn ) {
 172                 if ( mdb_txn_commit( txn ))
 173                         return -1;
 174                 txn = NULL;
 175         }
 176
 177         if( nholes ) {
 178                 unsigned i;
 179                 fprintf( stderr, "Error, entries missing!\n");
 180                 for (i=0; i<nholes; i++) {
 181                         fprintf(stderr, "  entry %ld: %s\n",
 182                                 holes[i].id, holes[i].dn.bv_val);
 183                 }
 184                 nholes = 0;
 185                 return -1;
 186         }
 187
 188         return 0;
 189 }
 190
 191 ID
 192 mdb_tool_entry_first_x(
 193         BackendDB *be,
 194         struct berval *base,
 195         int scope,
 196         Filter *f )
 197 {
 198         tool_base = base;
 199         tool_scope = scope;
 200         tool_filter = f;
 201
 202         return mdb_tool_entry_next( be );
 203 }
 204
 205 ID mdb_tool_entry_next(
 206         BackendDB *be )
 207 {
 208         int rc;
 209         ID id;
 210         struct mdb_info *mdb;
 211
 212         assert( be != NULL );
 213         assert( slapMode & SLAP_TOOL_MODE );
 214
 215         mdb = (struct mdb_info *) be->be_private;
 216         assert( mdb != NULL );
 217
 218         if ( !txn ) {
 219                 rc = mdb_txn_begin( mdb->mi_dbenv, NULL, MDB_RDONLY, &txn );
 220                 if ( rc )
 221                         return NOID;
 222                 rc = mdb_cursor_open( txn, mdb->mi_id2entry, &cursor );
 223                 if ( rc ) {
 224                         mdb_txn_abort( txn );
 225                         return NOID;
 226                 }
 227         }
 228
 229 next:;
 230         rc = mdb_cursor_get( cursor, &key, &data, MDB_NEXT );
 231
 232         if( rc ) {
 233                 return NOID;
 234         }
 235
 236         previd = *(ID *)key.mv_data;
 237         id = previd;
 238
 239         if ( tool_filter || tool_base ) {
 240                 static Operation op = {0};
 241                 static Opheader ohdr = {0};
 242
 243                 op.o_hdr = &ohdr;
 244                 op.o_bd = be;
 245                 op.o_tmpmemctx = NULL;
 246                 op.o_tmpmfuncs = &ch_mfuncs;
 247
 248                 if ( tool_next_entry ) {
 249                         mdb_entry_release( &op, tool_next_entry, 0 );
 250                         tool_next_entry = NULL;
 251                 }
 252
 253                 rc = mdb_tool_entry_get_int( be, id, &tool_next_entry );
 254                 if ( rc == LDAP_NO_SUCH_OBJECT ) {
 255                         goto next;
 256                 }
 257
 258                 assert( tool_next_entry != NULL );
 259
 260                 if ( tool_filter && test_filter( NULL, tool_next_entry, tool_filter ) != LDAP_COMPARE_TRUE )
 261                 {
 262                         mdb_entry_release( &op, tool_next_entry, 0 );
 263                         tool_next_entry = NULL;
 264                         goto next;
 265                 }
 266         }
 267
 268         return id;
 269 }
 270
 271 ID mdb_tool_dn2id_get(
 272         Backend *be,
 273         struct berval *dn
 274 )
 275 {
 276         struct mdb_info *mdb;
 277         Operation op = {0};
 278         Opheader ohdr = {0};
 279         ID id;
 280         int rc;
 281
 282         if ( BER_BVISEMPTY(dn) )
 283                 return 0;
 284
 285         mdb = (struct mdb_info *) be->be_private;
 286
 287         if ( !txn ) {
 288                 rc = mdb_txn_begin( mdb->mi_dbenv, NULL, (slapMode & SLAP_TOOL_READONLY) != 0 ?
 289                         MDB_RDONLY : 0, &txn );
 290                 if ( rc )
 291                         return NOID;
 292         }
 293
 294         op.o_hdr = &ohdr;
 295         op.o_bd = be;
 296         op.o_tmpmemctx = NULL;
 297         op.o_tmpmfuncs = &ch_mfuncs;
 298
 299         rc = mdb_dn2id( &op, txn, dn, &id, NULL, NULL );
 300         if ( rc == MDB_NOTFOUND )
 301                 return NOID;
 302
 303         return id;
 304 }
 305
 306 static int
 307 mdb_tool_entry_get_int( BackendDB *be, ID id, Entry **ep )
 308 {
 309         Operation op = {0};
 310         Opheader ohdr = {0};
 311
 312         Entry *e = NULL;
 313         struct berval dn = BER_BVNULL, ndn = BER_BVNULL;
 314         int rc;
 315
 316         assert( be != NULL );
 317         assert( slapMode & SLAP_TOOL_MODE );
 318
 319         if ( ( tool_filter || tool_base ) && id == previd && tool_next_entry != NULL ) {
 320                 *ep = tool_next_entry;
 321                 tool_next_entry = NULL;
 322                 return LDAP_SUCCESS;
 323         }
 324
 325         if ( id != previd ) {
 326                 key.mv_size = sizeof(ID);
 327                 key.mv_data = &id;
 328                 rc = mdb_cursor_get( cursor, &key, &data, MDB_SET );
 329                 if ( rc ) {
 330                         rc = LDAP_OTHER;
 331                         goto done;
 332                 }
 333         }
 334
 335         op.o_hdr = &ohdr;
 336         op.o_bd = be;
 337         op.o_tmpmemctx = NULL;
 338         op.o_tmpmfuncs = &ch_mfuncs;
 339         if ( slapMode & SLAP_TOOL_READONLY ) {
 340                 rc = mdb_id2name( &op, txn, &idcursor, id, &dn, &ndn );
 341                 if ( rc  ) {
 342                         rc = LDAP_OTHER;
 343                         mdb_entry_return( &op, e );
 344                         e = NULL;
 345                         goto done;
 346                 }
 347                 if ( tool_base != NULL ) {
 348                         if ( !dnIsSuffixScope( &ndn, tool_base, tool_scope ) ) {
 349                                 ch_free( dn.bv_val );
 350                                 ch_free( ndn.bv_val );
 351                                 rc = LDAP_NO_SUCH_OBJECT;
 352                         }
 353                 }
 354         }
 355         rc = mdb_entry_decode( &op, &data, &e );
 356         e->e_id = id;
 357         if ( !BER_BVISNULL( &dn )) {
 358                 e->e_name = dn;
 359                 e->e_nname = ndn;
 360         } else {
 361                 e->e_name.bv_val = NULL;
 362                 e->e_nname.bv_val = NULL;
 363         }
 364
 365 done:
 366         if ( e != NULL ) {
 367                 *ep = e;
 368         }
 369
 370         return rc;
 371 }
 372
 373 Entry*
 374 mdb_tool_entry_get( BackendDB *be, ID id )
 375 {
 376         Entry *e = NULL;
 377
 378         (void)mdb_tool_entry_get_int( be, id, &e );
 379         return e;
 380 }
 381
 382 static int mdb_tool_next_id(
 383         Operation *op,
 384         MDB_txn *tid,
 385         Entry *e,
 386         struct berval *text,
 387         int hole )
 388 {
 389         struct mdb_info *mdb = (struct mdb_info *) op->o_bd->be_private;
 390         struct berval dn = e->e_name;
 391         struct berval ndn = e->e_nname;
 392         struct berval pdn, npdn, nmatched;
 393         ID id, pid = 0;
 394         int rc;
 395
 396         if (ndn.bv_len == 0) {
 397                 e->e_id = 0;
 398                 return 0;
 399         }
 400
 401         rc = mdb_dn2id( op, tid, &ndn, &id, NULL, &nmatched );
 402         if ( rc == MDB_NOTFOUND ) {
 403                 if ( !be_issuffix( op->o_bd, &ndn ) ) {
 404                         ID eid = e->e_id;
 405                         dnParent( &ndn, &npdn );
 406                         if ( nmatched.bv_len != npdn.bv_len ) {
 407                                 dnParent( &dn, &pdn );
 408                                 e->e_name = pdn;
 409                                 e->e_nname = npdn;
 410                                 rc = mdb_tool_next_id( op, tid, e, text, 1 );
 411                                 e->e_name = dn;
 412                                 e->e_nname = ndn;
 413                                 if ( rc ) {
 414                                         return rc;
 415                                 }
 416                                 /* If parent didn't exist, it was created just now
 417                                  * and its ID is now in e->e_id. Make sure the current
 418                                  * entry gets added under the new parent ID.
 419                                  */
 420                                 if ( eid != e->e_id ) {
 421                                         pid = e->e_id;
 422                                 }
 423                         } else {
 424                                 pid = id;
 425                         }
 426                 }
 427                 rc = mdb_next_id( op->o_bd, tid, &e->e_id );
 428                 if ( rc ) {
 429                         snprintf( text->bv_val, text->bv_len,
 430                                 "next_id failed: %s (%d)",
 431                                 mdb_strerror(rc), rc );
 432                 Debug( LDAP_DEBUG_ANY,
 433                         "=> mdb_tool_next_id: %s\n", text->bv_val, 0, 0 );
 434                         return rc;
 435                 }
 436                 rc = mdb_dn2id_add( op, tid, pid, e );
 437                 if ( rc ) {
 438                         snprintf( text->bv_val, text->bv_len,
 439                                 "dn2id_add failed: %s (%d)",
 440                                 mdb_strerror(rc), rc );
 441                         Debug( LDAP_DEBUG_ANY,
 442                                 "=> mdb_tool_next_id: %s\n", text->bv_val, 0, 0 );
 443                 } else if ( hole ) {
 444                         MDB_val key, data;
 445                         if ( nholes == nhmax - 1 ) {
 446                                 if ( holes == hbuf ) {
 447                                         holes = ch_malloc( nhmax * sizeof(dn_id) * 2 );
 448                                         AC_MEMCPY( holes, hbuf, sizeof(hbuf) );
 449                                 } else {
 450                                         holes = ch_realloc( holes, nhmax * sizeof(dn_id) * 2 );
 451                                 }
 452                                 nhmax *= 2;
 453                         }
 454                         ber_dupbv( &holes[nholes].dn, &ndn );
 455                         holes[nholes++].id = e->e_id;
 456                         key.mv_size = sizeof(ID);
 457                         key.mv_data = &e->e_id;
 458                         data.mv_size = 0;
 459                         data.mv_data = NULL;
 460                         rc = mdb_put( tid, mdb->mi_id2entry, &key, &data, MDB_NOOVERWRITE );
 461                         if ( rc == MDB_KEYEXIST )
 462                                 rc = 0;
 463                         if ( rc ) {
 464                                 snprintf( text->bv_val, text->bv_len,
 465                                         "dummy id2entry add failed: %s (%d)",
 466                                         mdb_strerror(rc), rc );
 467                                 Debug( LDAP_DEBUG_ANY,
 468                                         "=> mdb_tool_next_id: %s\n", text->bv_val, 0, 0 );
 469                         }
 470                 }
 471         } else if ( !hole ) {
 472                 unsigned i, j;
 473
 474                 e->e_id = id;
 475
 476                 for ( i=0; i<nholes; i++) {
 477                         if ( holes[i].id == e->e_id ) {
 478                                 free(holes[i].dn.bv_val);
 479                                 for (j=i;j<nholes;j++) holes[j] = holes[j+1];
 480                                 holes[j].id = 0;
 481                                 nholes--;
 482                                 break;
 483                         } else if ( holes[i].id > e->e_id ) {
 484                                 break;
 485                         }
 486                 }
 487         }
 488         return rc;
 489 }
 490
 491 static int
 492 mdb_tool_index_add(
 493         Operation *op,
 494         MDB_txn *txn,
 495         Entry *e )
 496 {
 497         struct mdb_info *mdb = (struct mdb_info *) op->o_bd->be_private;
 498
 499         if ( !mdb->mi_nattrs )
 500                 return 0;
 501
 502 #if 0
 503         if ( slapMode & SLAP_TOOL_QUICK ) {
 504                 IndexRec *ir;
 505                 int i, rc;
 506                 Attribute *a;
 507
 508                 ir = mdb_tool_index_rec;
 509                 memset(ir, 0, mdb->bi_nattrs * sizeof( IndexRec ));
 510
 511                 for ( a = e->e_attrs; a != NULL; a = a->a_next ) {
 512                         rc = mdb_index_recset( mdb, a, a->a_desc->ad_type,
 513                                 &a->a_desc->ad_tags, ir );
 514                         if ( rc )
 515                                 return rc;
 516                 }
 517                 mdb_tool_ix_id = e->e_id;
 518                 mdb_tool_ix_op = op;
 519                 ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex );
 520                 /* Wait for all threads to be ready */
 521                 while ( mdb_tool_index_tcount ) {
 522                         ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main,
 523                                 &mdb_tool_index_mutex );
 524                 }
 525                 for ( i=1; i<slap_tool_thread_max; i++ )
 526                         mdb_tool_index_threads[i] = LDAP_BUSY;
 527                 mdb_tool_index_tcount = slap_tool_thread_max - 1;
 528                 ldap_pvt_thread_cond_broadcast( &mdb_tool_index_cond_work );
 529                 ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex );
 530                 rc = mdb_index_recrun( op, mdb, ir, e->e_id, 0 );
 531                 if ( rc )
 532                         return rc;
 533                 ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex );
 534                 for ( i=1; i<slap_tool_thread_max; i++ ) {
 535                         if ( mdb_tool_index_threads[i] == LDAP_BUSY ) {
 536                                 ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main,
 537                                         &mdb_tool_index_mutex );
 538                                 i--;
 539                                 continue;
 540                         }
 541                         if ( mdb_tool_index_threads[i] ) {
 542                                 rc = mdb_tool_index_threads[i];
 543                                 break;
 544                         }
 545                 }
 546                 ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex );
 547                 return rc;
 548         } else
 549 #endif
 550         {
 551                 return mdb_index_entry_add( op, txn, e );
 552         }
 553 }
 554
 555 ID mdb_tool_entry_put(
 556         BackendDB *be,
 557         Entry *e,
 558         struct berval *text )
 559 {
 560         int rc;
 561         struct mdb_info *mdb;
 562         Operation op = {0};
 563         Opheader ohdr = {0};
 564
 565         assert( be != NULL );
 566         assert( slapMode & SLAP_TOOL_MODE );
 567
 568         assert( text != NULL );
 569         assert( text->bv_val != NULL );
 570         assert( text->bv_val[0] == '\0' );      /* overconservative? */
 571
 572         Debug( LDAP_DEBUG_TRACE, "=> " LDAP_XSTRING(mdb_tool_entry_put)
 573                 "( %ld, \"%s\" )\n", (long) e->e_id, e->e_dn, 0 );
 574
 575         mdb = (struct mdb_info *) be->be_private;
 576
 577         if ( !txn ) {
 578         rc = mdb_txn_begin( mdb->mi_dbenv, NULL, 0, &txn );
 579         if( rc != 0 ) {
 580                 snprintf( text->bv_val, text->bv_len,
 581                         "txn_begin failed: %s (%d)",
 582                         mdb_strerror(rc), rc );
 583                 Debug( LDAP_DEBUG_ANY,
 584                         "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
 585                          text->bv_val, 0, 0 );
 586                 return NOID;
 587         }
 588         }
 589
 590         op.o_hdr = &ohdr;
 591         op.o_bd = be;
 592         op.o_tmpmemctx = NULL;
 593         op.o_tmpmfuncs = &ch_mfuncs;
 594
 595         /* add dn2id indices */
 596         rc = mdb_tool_next_id( &op, txn, e, text, 0 );
 597         if( rc != 0 ) {
 598                 goto done;
 599         }
 600
 601         rc = mdb_tool_index_add( &op, txn, e );
 602         if( rc != 0 ) {
 603                 snprintf( text->bv_val, text->bv_len,
 604                                 "index_entry_add failed: err=%d", rc );
 605                 Debug( LDAP_DEBUG_ANY,
 606                         "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
 607                         text->bv_val, 0, 0 );
 608                 goto done;
 609         }
 610
 611
 612         /* id2entry index */
 613         rc = mdb_id2entry_add( &op, txn, e );
 614         if( rc != 0 ) {
 615                 snprintf( text->bv_val, text->bv_len,
 616                                 "id2entry_add failed: err=%d", rc );
 617                 Debug( LDAP_DEBUG_ANY,
 618                         "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
 619                         text->bv_val, 0, 0 );
 620                 goto done;
 621         }
 622
 623 done:
 624         if( rc == 0 ) {
 625                 mdb_writes++;
 626                 if ( mdb_writes >= mdb_writes_per_commit ) {
 627                         MDB_TOOL_IDL_FLUSH( be, txn );
 628                         rc = mdb_txn_commit( txn );
 629                         mdb_writes = 0;
 630                         txn = NULL;
 631                         if( rc != 0 ) {
 632                                 snprintf( text->bv_val, text->bv_len,
 633                                                 "txn_commit failed: %s (%d)",
 634                                                 mdb_strerror(rc), rc );
 635                                 Debug( LDAP_DEBUG_ANY,
 636                                         "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
 637                                         text->bv_val, 0, 0 );
 638                                 e->e_id = NOID;
 639                         }
 640                 }
 641
 642         } else {
 643                 mdb_txn_abort( txn );
 644                 txn = NULL;
 645                 snprintf( text->bv_val, text->bv_len,
 646                         "txn_aborted! %s (%d)",
 647                         rc == LDAP_OTHER ? "Internal error" :
 648                         mdb_strerror(rc), rc );
 649                 Debug( LDAP_DEBUG_ANY,
 650                         "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
 651                         text->bv_val, 0, 0 );
 652                 e->e_id = NOID;
 653         }
 654
 655         return e->e_id;
 656 }
 657
 658 int mdb_tool_entry_reindex(
 659         BackendDB *be,
 660         ID id,
 661         AttributeDescription **adv )
 662 {
 663         struct mdb_info *mi = (struct mdb_info *) be->be_private;
 664         int rc;
 665         Entry *e;
 666         Operation op = {0};
 667         Opheader ohdr = {0};
 668
 669         Debug( LDAP_DEBUG_ARGS,
 670                 "=> " LDAP_XSTRING(mdb_tool_entry_reindex) "( %ld )\n",
 671                 (long) id, 0, 0 );
 672         assert( tool_base == NULL );
 673         assert( tool_filter == NULL );
 674
 675         /* No indexes configured, nothing to do. Could return an
 676          * error here to shortcut things.
 677          */
 678         if (!mi->mi_attrs) {
 679                 return 0;
 680         }
 681
 682         /* Check for explicit list of attrs to index */
 683         if ( adv ) {
 684                 int i, j, n;
 685
 686                 if ( mi->mi_attrs[0]->ai_desc != adv[0] ) {
 687                         /* count */
 688                         for ( n = 0; adv[n]; n++ ) ;
 689
 690                         /* insertion sort */
 691                         for ( i = 0; i < n; i++ ) {
 692                                 AttributeDescription *ad = adv[i];
 693                                 for ( j = i-1; j>=0; j--) {
 694                                         if ( SLAP_PTRCMP( adv[j], ad ) <= 0 ) break;
 695                                         adv[j+1] = adv[j];
 696                                 }
 697                                 adv[j+1] = ad;
 698                         }
 699                 }
 700
 701                 for ( i = 0; adv[i]; i++ ) {
 702                         if ( mi->mi_attrs[i]->ai_desc != adv[i] ) {
 703                                 for ( j = i+1; j < mi->mi_nattrs; j++ ) {
 704                                         if ( mi->mi_attrs[j]->ai_desc == adv[i] ) {
 705                                                 AttrInfo *ai = mi->mi_attrs[i];
 706                                                 mi->mi_attrs[i] = mi->mi_attrs[j];
 707                                                 mi->mi_attrs[j] = ai;
 708                                                 break;
 709                                         }
 710                                 }
 711                                 if ( j == mi->mi_nattrs ) {
 712                                         Debug( LDAP_DEBUG_ANY,
 713                                                 LDAP_XSTRING(mdb_tool_entry_reindex)
 714                                                 ": no index configured for %s\n",
 715                                                 adv[i]->ad_cname.bv_val, 0, 0 );
 716                                         return -1;
 717                                 }
 718                         }
 719                 }
 720                 mi->mi_nattrs = i;
 721         }
 722
 723         if ( slapMode & SLAP_TRUNCATE_MODE ) {
 724                 int i;
 725                 for ( i=0; i < mi->mi_nattrs; i++ ) {
 726                         rc = mdb_drop( txn, mi->mi_attrs[i]->ai_dbi, 0 );
 727                         if ( rc ) {
 728                                 Debug( LDAP_DEBUG_ANY,
 729                                         LDAP_XSTRING(mdb_tool_entry_reindex)
 730                                         ": (Truncate) mdb_drop(%s) failed: %s (%d)\n",
 731                                         mi->mi_attrs[i]->ai_desc->ad_type->sat_cname.bv_val,
 732                                         mdb_strerror(rc), rc );
 733                                 return -1;
 734                         }
 735                 }
 736                 slapMode ^= SLAP_TRUNCATE_MODE;
 737         }
 738
 739         e = mdb_tool_entry_get( be, id );
 740
 741         if( e == NULL ) {
 742                 Debug( LDAP_DEBUG_ANY,
 743                         LDAP_XSTRING(mdb_tool_entry_reindex)
 744                         ": could not locate id=%ld\n",
 745                         (long) id, 0, 0 );
 746                 return -1;
 747         }
 748
 749         if ( !txi ) {
 750                 rc = mdb_txn_begin( mi->mi_dbenv, NULL, 0, &txi );
 751                 if( rc != 0 ) {
 752                         Debug( LDAP_DEBUG_ANY,
 753                                 "=> " LDAP_XSTRING(mdb_tool_entry_reindex) ": "
 754                                 "txn_begin failed: %s (%d)\n",
 755                                 mdb_strerror(rc), rc, 0 );
 756                         goto done;
 757                 }
 758         }
 759
 760         /*
 761          * just (re)add them for now
 762          * assume that some other routine (not yet implemented)
 763          * will zap index databases
 764          *
 765          */
 766
 767         Debug( LDAP_DEBUG_TRACE,
 768                 "=> " LDAP_XSTRING(mdb_tool_entry_reindex) "( %ld, \"%s\" )\n",
 769                 (long) id, e->e_dn, 0 );
 770
 771         op.o_hdr = &ohdr;
 772         op.o_bd = be;
 773         op.o_tmpmemctx = NULL;
 774         op.o_tmpmfuncs = &ch_mfuncs;
 775
 776         rc = mdb_tool_index_add( &op, txi, e );
 777
 778 done:
 779         if( rc == 0 ) {
 780                 mdb_writes++;
 781                 if ( mdb_writes >= mdb_writes_per_commit ) {
 782                         MDB_TOOL_IDL_FLUSH( be, txi );
 783                         rc = mdb_txn_commit( txi );
 784                         if( rc != 0 ) {
 785                                 Debug( LDAP_DEBUG_ANY,
 786                                         "=> " LDAP_XSTRING(mdb_tool_entry_reindex)
 787                                         ": txn_commit failed: %s (%d)\n",
 788                                         mdb_strerror(rc), rc, 0 );
 789                                 e->e_id = NOID;
 790                         }
 791                         txi = NULL;
 792                 }
 793
 794         } else {
 795                 mdb_txn_abort( txi );
 796                 Debug( LDAP_DEBUG_ANY,
 797                         "=> " LDAP_XSTRING(mdb_tool_entry_reindex)
 798                         ": txn_aborted! err=%d\n",
 799                         rc, 0, 0 );
 800                 e->e_id = NOID;
 801                 txi = NULL;
 802         }
 803         mdb_entry_release( &op, e, 0 );
 804
 805         return rc;
 806 }
 807
 808 ID mdb_tool_entry_modify(
 809         BackendDB *be,
 810         Entry *e,
 811         struct berval *text )
 812 {
 813         int rc;
 814         struct mdb_info *mdb;
 815         MDB_txn *tid;
 816         Operation op = {0};
 817         Opheader ohdr = {0};
 818
 819         assert( be != NULL );
 820         assert( slapMode & SLAP_TOOL_MODE );
 821
 822         assert( text != NULL );
 823         assert( text->bv_val != NULL );
 824         assert( text->bv_val[0] == '\0' );      /* overconservative? */
 825
 826         assert ( e->e_id != NOID );
 827
 828         Debug( LDAP_DEBUG_TRACE,
 829                 "=> " LDAP_XSTRING(mdb_tool_entry_modify) "( %ld, \"%s\" )\n",
 830                 (long) e->e_id, e->e_dn, 0 );
 831
 832         mdb = (struct mdb_info *) be->be_private;
 833
 834         if( cursor ) {
 835                 mdb_cursor_close( cursor );
 836                 cursor = NULL;
 837         }
 838         rc = mdb_txn_begin( mdb->mi_dbenv, NULL, 0, &tid );
 839         if( rc != 0 ) {
 840                 snprintf( text->bv_val, text->bv_len,
 841                         "txn_begin failed: %s (%d)",
 842                         mdb_strerror(rc), rc );
 843                 Debug( LDAP_DEBUG_ANY,
 844                         "=> " LDAP_XSTRING(mdb_tool_entry_modify) ": %s\n",
 845                          text->bv_val, 0, 0 );
 846                 return NOID;
 847         }
 848
 849         op.o_hdr = &ohdr;
 850         op.o_bd = be;
 851         op.o_tmpmemctx = NULL;
 852         op.o_tmpmfuncs = &ch_mfuncs;
 853
 854         /* id2entry index */
 855         rc = mdb_id2entry_update( &op, tid, e );
 856         if( rc != 0 ) {
 857                 snprintf( text->bv_val, text->bv_len,
 858                                 "id2entry_update failed: err=%d", rc );
 859                 Debug( LDAP_DEBUG_ANY,
 860                         "=> " LDAP_XSTRING(mdb_tool_entry_modify) ": %s\n",
 861                         text->bv_val, 0, 0 );
 862                 goto done;
 863         }
 864
 865 done:
 866         if( rc == 0 ) {
 867                 rc = mdb_txn_commit( tid );
 868                 if( rc != 0 ) {
 869                         snprintf( text->bv_val, text->bv_len,
 870                                         "txn_commit failed: %s (%d)",
 871                                         mdb_strerror(rc), rc );
 872                         Debug( LDAP_DEBUG_ANY,
 873                                 "=> " LDAP_XSTRING(mdb_tool_entry_modify) ": "
 874                                 "%s\n", text->bv_val, 0, 0 );
 875                         e->e_id = NOID;
 876                 }
 877
 878         } else {
 879                 mdb_txn_abort( tid );
 880                 snprintf( text->bv_val, text->bv_len,
 881                         "txn_aborted! %s (%d)",
 882                         mdb_strerror(rc), rc );
 883                 Debug( LDAP_DEBUG_ANY,
 884                         "=> " LDAP_XSTRING(mdb_tool_entry_modify) ": %s\n",
 885                         text->bv_val, 0, 0 );
 886                 e->e_id = NOID;
 887         }
 888
 889         return e->e_id;
 890 }
 891
 892 #if 0
 893 static void *
 894 mdb_tool_index_task( void *ctx, void *ptr )
 895 {
 896         int base = *(int *)ptr;
 897
 898         free( ptr );
 899         while ( 1 ) {
 900                 ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex );
 901                 mdb_tool_index_tcount--;
 902                 if ( !mdb_tool_index_tcount )
 903                         ldap_pvt_thread_cond_signal( &mdb_tool_index_cond_main );
 904                 ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_work,
 905                         &mdb_tool_index_mutex );
 906                 if ( slapd_shutdown ) {
 907                         mdb_tool_index_tcount--;
 908                         if ( !mdb_tool_index_tcount )
 909                                 ldap_pvt_thread_cond_signal( &mdb_tool_index_cond_main );
 910                         ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex );
 911                         break;
 912                 }
 913                 ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex );
 914
 915                 mdb_tool_index_threads[base] = mdb_index_recrun( mdb_tool_ix_op,
 916                         mdb_tool_info, mdb_tool_index_rec, mdb_tool_ix_id, base );
 917         }
 918
 919         return NULL;
 920 }
 921 #endif
 922
 923 #ifdef MDB_TOOL_IDL_CACHING
 924 static int
 925 mdb_tool_idl_cmp( const void *v1, const void *v2 )
 926 {
 927         const mdb_tool_idl_cache *c1 = v1, *c2 = v2;
 928         int rc;
 929
 930         if (( rc = c1->kstr.bv_len - c2->kstr.bv_len )) return rc;
 931         return memcmp( c1->kstr.bv_val, c2->kstr.bv_val, c1->kstr.bv_len );
 932 }
 933
 934 static int
 935 mdb_tool_idl_flush_one( MDB_cursor *mc, mdb_tool_idl_cache *ic )
 936 {
 937         mdb_tool_idl_cache_entry *ice;
 938         MDB_val key, data[2];
 939         int i, rc;
 940         ID id, nid;
 941
 942         /* Freshly allocated, ignore it */
 943         if ( !ic->head && ic->count <= MDB_IDL_DB_SIZE ) {
 944                 return 0;
 945         }
 946
 947         key.mv_data = ic->kstr.bv_val;
 948         key.mv_size = ic->kstr.bv_len;
 949
 950         if ( ic->count > MDB_IDL_DB_SIZE ) {
 951                 while ( ic->flags & WAS_FOUND ) {
 952                         rc = mdb_cursor_get( mc, &key, data, MDB_SET );
 953                         if ( rc ) {
 954                                 /* FIXME: find out why this happens */
 955                                 ic->flags = 0;
 956                                 break;
 957                         }
 958                         if ( ic->flags & WAS_RANGE ) {
 959                                 /* Skip lo */
 960                                 rc = mdb_cursor_get( mc, &key, data, MDB_NEXT_DUP );
 961
 962                                 /* Get hi */
 963                                 rc = mdb_cursor_get( mc, &key, data, MDB_NEXT_DUP );
 964
 965                                 /* Store range hi */
 966                                 data[0].mv_data = &ic->last;
 967                                 rc = mdb_cursor_put( mc, &key, data, MDB_CURRENT );
 968                         } else {
 969                                 /* Delete old data, replace with range */
 970                                 ic->first = *(ID *)data[0].mv_data;
 971                                 mdb_cursor_del( mc, MDB_NODUPDATA );
 972                         }
 973                         break;
 974                 }
 975                 if ( !(ic->flags & WAS_RANGE)) {
 976                         /* range, didn't exist before */
 977                         nid = 0;
 978                         data[0].mv_size = sizeof(ID);
 979                         data[0].mv_data = &nid;
 980                         rc = mdb_cursor_put( mc, &key, data, 0 );
 981                         if ( rc == 0 ) {
 982                                 data[0].mv_data = &ic->first;
 983                                 rc = mdb_cursor_put( mc, &key, data, 0 );
 984                                 if ( rc == 0 ) {
 985                                         data[0].mv_data = &ic->last;
 986                                         rc = mdb_cursor_put( mc, &key, data, 0 );
 987                                 }
 988                         }
 989                         if ( rc ) {
 990                                 rc = -1;
 991                         }
 992                 }
 993         } else {
 994                 /* Normal write */
 995                 int n;
 996
 997                 data[0].mv_size = sizeof(ID);
 998                 rc = 0;
 999                 i = ic->offset;
1000                 for ( ice = ic->head, n=0; ice; ice = ice->next, n++ ) {
1001                         int end;
1002                         if ( ice->next ) {
1003                                 end = IDBLOCK;
1004                         } else {
1005                                 end = ic->count & (IDBLOCK-1);
1006                                 if ( !end )
1007                                         end = IDBLOCK;
1008                         }
1009                         data[1].mv_size = end - i;
1010                         data[0].mv_data = &ice->ids[i];
1011                         i = 0;
1012                         rc = mdb_cursor_put( mc, &key, data, MDB_NODUPDATA|MDB_APPEND|MDB_MULTIPLE );
1013                         if ( rc ) {
1014                                 if ( rc == MDB_KEYEXIST ) {
1015                                         rc = 0;
1016                                         continue;
1017                                 }
1018                                 rc = -1;
1019                                 break;
1020                         }
1021                 }
1022                 if ( ic->head ) {
1023                         ic->tail->next = mdb_tool_idl_free_list;
1024                         mdb_tool_idl_free_list = ic->head;
1025                 }
1026         }
1027         ch_free( ic );
1028         return rc;
1029 }
1030
1031 static int
1032 mdb_tool_idl_flush_db( MDB_txn *txn, MDB_dbi dbi, Avlnode *root )
1033 {
1034         MDB_cursor *mc;
1035         int rc;
1036
1037         mdb_cursor_open( txn, dbi, &mc );
1038         root = tavl_end( root, TAVL_DIR_LEFT );
1039         do {
1040                 rc = mdb_tool_idl_flush_one( mc, root->avl_data );
1041                 if ( rc != -1 )
1042                         rc = 0;
1043         } while ((root = tavl_next(root, TAVL_DIR_RIGHT)));
1044         mdb_cursor_close( mc );
1045
1046         return rc;
1047 }
1048
1049 static int
1050 mdb_tool_idl_flush( BackendDB *be, MDB_txn *txn )
1051 {
1052         struct mdb_info *mdb = (struct mdb_info *) be->be_private;
1053         int rc = 0;
1054         unsigned int i, dbi;
1055
1056         for ( i=0; i < mdb->mi_nattrs; i++ ) {
1057                 dbi = mdb->mi_attrs[i]->ai_dbi;
1058                 if ( !mdb_tool_roots[dbi] ) continue;
1059                 rc = mdb_tool_idl_flush_db( txn, dbi, mdb_tool_roots[dbi] );
1060                 tavl_free(mdb_tool_roots[dbi], NULL);
1061                 mdb_tool_roots[dbi] = NULL;
1062                 if ( rc )
1063                         break;
1064         }
1065         return rc;
1066 }
1067
1068 int mdb_tool_idl_add(
1069         MDB_cursor *mc,
1070         struct berval *keys,
1071         ID id )
1072 {
1073         MDB_dbi dbi;
1074         mdb_tool_idl_cache *ic, itmp;
1075         mdb_tool_idl_cache_entry *ice;
1076         int i, rc, lcount;
1077
1078         dbi = mdb_cursor_dbi(mc);
1079         for (i=0; keys[i].bv_val; i++) {
1080         itmp.kstr = keys[i];
1081         ic = tavl_find( (Avlnode *)mdb_tool_roots[dbi], &itmp, mdb_tool_idl_cmp );
1082
1083         /* No entry yet, create one */
1084         if ( !ic ) {
1085                 MDB_val key, data;
1086                 ID nid;
1087                 int rc;
1088
1089                 ic = ch_malloc( sizeof( mdb_tool_idl_cache ) + itmp.kstr.bv_len );
1090                 ic->kstr.bv_len = itmp.kstr.bv_len;
1091                 ic->kstr.bv_val = (char *)(ic+1);
1092                 memcpy( ic->kstr.bv_val, itmp.kstr.bv_val, ic->kstr.bv_len );
1093                 ic->head = ic->tail = NULL;
1094                 ic->last = 0;
1095                 ic->count = 0;
1096                 ic->offset = 0;
1097                 ic->flags = 0;
1098                 tavl_insert( (Avlnode **)&mdb_tool_roots[dbi], ic, mdb_tool_idl_cmp,
1099                         avl_dup_error );
1100
1101                 /* load existing key count here */
1102                 key.mv_size = keys[i].bv_len;
1103                 key.mv_data = keys[i].bv_val;
1104                 rc = mdb_cursor_get( mc, &key, &data, MDB_SET );
1105                 if ( rc == 0 ) {
1106                         ic->flags |= WAS_FOUND;
1107                         nid = *(ID *)data.mv_data;
1108                         if ( nid == 0 ) {
1109                                 ic->count = MDB_IDL_DB_SIZE+1;
1110                                 ic->flags |= WAS_RANGE;
1111                         } else {
1112                                 size_t count;
1113
1114                                 mdb_cursor_count( mc, &count );
1115                                 ic->count = count;
1116                                 ic->first = nid;
1117                                 ic->offset = count & (IDBLOCK-1);
1118                         }
1119                 }
1120         }
1121         /* are we a range already? */
1122         if ( ic->count > MDB_IDL_DB_SIZE ) {
1123                 ic->last = id;
1124                 continue;
1125         /* Are we at the limit, and converting to a range? */
1126         } else if ( ic->count == MDB_IDL_DB_SIZE ) {
1127                 int n;
1128                 for ( ice = ic->head, n=0; ice; ice = ice->next, n++ )
1129                         /* counting */ ;
1130                 if ( n ) {
1131                         ic->tail->next = mdb_tool_idl_free_list;
1132                         mdb_tool_idl_free_list = ic->head;
1133                 }
1134                 ic->head = ic->tail = NULL;
1135                 ic->last = id;
1136                 ic->count++;
1137                 continue;
1138         }
1139         /* No free block, create that too */
1140         lcount = ic->count & (IDBLOCK-1);
1141         if ( !ic->tail || lcount == 0) {
1142                 ice = NULL;
1143                 if ( mdb_tool_idl_free_list ) {
1144                         ice = mdb_tool_idl_free_list;
1145                         mdb_tool_idl_free_list = ice->next;
1146                 }
1147                 if ( !ice ) {
1148                         ice = ch_malloc( sizeof( mdb_tool_idl_cache_entry ));
1149                 }
1150                 ice->next = NULL;
1151                 if ( !ic->head ) {
1152                         ic->head = ice;
1153                 } else {
1154                         ic->tail->next = ice;
1155                 }
1156                 ic->tail = ice;
1157                 if ( lcount )
1158                         ice->ids[lcount-1] = 0;
1159                 if ( !ic->count )
1160                         ic->first = id;
1161         }
1162         ice = ic->tail;
1163         if (!lcount || ice->ids[lcount-1] != id)
1164                 ice->ids[lcount] = id;
1165         ic->count++;
1166         }
1167
1168         return 0;
1169 }
1170 #endif /* MDB_TOOL_IDL_CACHING */