git.sur5r.net Git - openldap/blob - servers/slapd/back-mdb/tools.c

   1 /* tools.c - tools for slap tools */
   2 /* $OpenLDAP$ */
   3 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
   4  *
   5  * Copyright 2011 The OpenLDAP Foundation.
   6  * All rights reserved.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted only as authorized by the OpenLDAP
  10  * Public License.
  11  *
  12  * A copy of this license is available in the file LICENSE in the
  13  * top-level directory of the distribution or, alternatively, at
  14  * <http://www.OpenLDAP.org/license.html>.
  15  */
  16
  17 #include "portable.h"
  18
  19 #include <stdio.h>
  20 #include <ac/string.h>
  21 #include <ac/errno.h>
  22
  23 #define AVL_INTERNAL
  24 #include "back-mdb.h"
  25 #include "idl.h"
  26
  27 static int mdb_tool_idl_flush( BackendDB *be, MDB_txn *txn );
  28
  29 #define IDBLOCK 1024
  30
  31 typedef struct mdb_tool_idl_cache_entry {
  32         struct mdb_tool_idl_cache_entry *next;
  33         ID ids[IDBLOCK];
  34 } mdb_tool_idl_cache_entry;
  35
  36 typedef struct mdb_tool_idl_cache {
  37         struct berval kstr;
  38         mdb_tool_idl_cache_entry *head, *tail;
  39         ID first, last;
  40         int count;
  41         short offset;
  42         short flags;
  43 } mdb_tool_idl_cache;
  44 #define WAS_FOUND       0x01
  45 #define WAS_RANGE       0x02
  46
  47 static mdb_tool_idl_cache_entry *mdb_tool_idl_free_list;
  48 static Avlnode *mdb_tool_roots[MDB_INDICES];
  49
  50 static MDB_txn *txn = NULL, *txi = NULL;
  51 static MDB_cursor *cursor = NULL, *idcursor = NULL;
  52 static MDB_val key, data;
  53 static ID previd = NOID;
  54
  55 typedef struct dn_id {
  56         ID id;
  57         struct berval dn;
  58 } dn_id;
  59
  60 #define HOLE_SIZE       4096
  61 static dn_id hbuf[HOLE_SIZE], *holes = hbuf;
  62 static unsigned nhmax = HOLE_SIZE;
  63 static unsigned nholes;
  64
  65 static struct berval    *tool_base;
  66 static int              tool_scope;
  67 static Filter           *tool_filter;
  68 static Entry            *tool_next_entry;
  69
  70 #if 0
  71 static ID mdb_tool_ix_id;
  72 static Operation *mdb_tool_ix_op;
  73 static int *mdb_tool_index_threads, mdb_tool_index_tcount;
  74 static void *mdb_tool_index_rec;
  75 static struct mdb_info *mdb_tool_info;
  76 static ldap_pvt_thread_mutex_t mdb_tool_index_mutex;
  77 static ldap_pvt_thread_cond_t mdb_tool_index_cond_main;
  78 static ldap_pvt_thread_cond_t mdb_tool_index_cond_work;
  79 static void * mdb_tool_index_task( void *ctx, void *ptr );
  80 #endif
  81
  82 static int      mdb_writes, mdb_writes_per_commit;
  83
  84 static int
  85 mdb_tool_entry_get_int( BackendDB *be, ID id, Entry **ep );
  86
  87 int mdb_tool_entry_open(
  88         BackendDB *be, int mode )
  89 {
  90         /* In Quick mode, commit once per 1000 entries */
  91         mdb_writes = 0;
  92         if ( slapMode & SLAP_TOOL_QUICK )
  93                 mdb_writes_per_commit = 1000;
  94         else
  95                 mdb_writes_per_commit = 1;
  96
  97 #if 0
  98         /* Set up for threaded slapindex */
  99         if (( slapMode & (SLAP_TOOL_QUICK|SLAP_TOOL_READONLY)) == SLAP_TOOL_QUICK ) {
 100                 if ( !mdb_tool_info ) {
 101                         ldap_pvt_thread_mutex_init( &mdb_tool_index_mutex );
 102                         ldap_pvt_thread_cond_init( &mdb_tool_index_cond_main );
 103                         ldap_pvt_thread_cond_init( &mdb_tool_index_cond_work );
 104                         if ( mdb->bi_nattrs ) {
 105                                 int i;
 106                                 mdb_tool_index_threads = ch_malloc( slap_tool_thread_max * sizeof( int ));
 107                                 mdb_tool_index_rec = ch_malloc( mdb->bi_nattrs * sizeof( IndexRec ));
 108                                 mdb_tool_index_tcount = slap_tool_thread_max - 1;
 109                                 for (i=1; i<slap_tool_thread_max; i++) {
 110                                         int *ptr = ch_malloc( sizeof( int ));
 111                                         *ptr = i;
 112                                         ldap_pvt_thread_pool_submit( &connection_pool,
 113                                                 mdb_tool_index_task, ptr );
 114                                 }
 115                         }
 116                         mdb_tool_info = mdb;
 117                 }
 118         }
 119 #endif
 120
 121         return 0;
 122 }
 123
 124 int mdb_tool_entry_close(
 125         BackendDB *be )
 126 {
 127 #if 0
 128         if ( mdb_tool_info ) {
 129                 slapd_shutdown = 1;
 130                 ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex );
 131
 132                 /* There might still be some threads starting */
 133                 while ( mdb_tool_index_tcount ) {
 134                         ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main,
 135                                         &mdb_tool_index_mutex );
 136                 }
 137
 138                 mdb_tool_index_tcount = slap_tool_thread_max - 1;
 139                 ldap_pvt_thread_cond_broadcast( &mdb_tool_index_cond_work );
 140
 141                 /* Make sure all threads are stopped */
 142                 while ( mdb_tool_index_tcount ) {
 143                         ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main,
 144                                 &mdb_tool_index_mutex );
 145                 }
 146                 ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex );
 147
 148                 mdb_tool_info = NULL;
 149                 slapd_shutdown = 0;
 150                 ch_free( mdb_tool_index_threads );
 151                 ch_free( mdb_tool_index_rec );
 152                 mdb_tool_index_tcount = slap_tool_thread_max - 1;
 153         }
 154 #endif
 155
 156         if( idcursor ) {
 157                 mdb_cursor_close( idcursor );
 158                 idcursor = NULL;
 159         }
 160         if( cursor ) {
 161                 mdb_cursor_close( cursor );
 162                 cursor = NULL;
 163         }
 164         mdb_tool_idl_flush( be, txn );
 165         if( txn ) {
 166                 if ( mdb_txn_commit( txn ))
 167                         return -1;
 168                 txn = NULL;
 169         }
 170
 171         if( nholes ) {
 172                 unsigned i;
 173                 fprintf( stderr, "Error, entries missing!\n");
 174                 for (i=0; i<nholes; i++) {
 175                         fprintf(stderr, "  entry %ld: %s\n",
 176                                 holes[i].id, holes[i].dn.bv_val);
 177                 }
 178                 nholes = 0;
 179                 return -1;
 180         }
 181
 182         return 0;
 183 }
 184
 185 ID
 186 mdb_tool_entry_first_x(
 187         BackendDB *be,
 188         struct berval *base,
 189         int scope,
 190         Filter *f )
 191 {
 192         tool_base = base;
 193         tool_scope = scope;
 194         tool_filter = f;
 195
 196         return mdb_tool_entry_next( be );
 197 }
 198
 199 ID mdb_tool_entry_next(
 200         BackendDB *be )
 201 {
 202         int rc;
 203         ID id;
 204         struct mdb_info *mdb;
 205
 206         assert( be != NULL );
 207         assert( slapMode & SLAP_TOOL_MODE );
 208
 209         mdb = (struct mdb_info *) be->be_private;
 210         assert( mdb != NULL );
 211
 212         if ( !txn ) {
 213                 rc = mdb_txn_begin( mdb->mi_dbenv, NULL, MDB_RDONLY, &txn );
 214                 if ( rc )
 215                         return NOID;
 216                 rc = mdb_cursor_open( txn, mdb->mi_id2entry, &cursor );
 217                 if ( rc ) {
 218                         mdb_txn_abort( txn );
 219                         return NOID;
 220                 }
 221         }
 222
 223 next:;
 224         rc = mdb_cursor_get( cursor, &key, &data, MDB_NEXT );
 225
 226         if( rc ) {
 227                 return NOID;
 228         }
 229
 230         previd = *(ID *)key.mv_data;
 231         id = previd;
 232
 233         if ( tool_filter || tool_base ) {
 234                 static Operation op = {0};
 235                 static Opheader ohdr = {0};
 236
 237                 op.o_hdr = &ohdr;
 238                 op.o_bd = be;
 239                 op.o_tmpmemctx = NULL;
 240                 op.o_tmpmfuncs = &ch_mfuncs;
 241
 242                 if ( tool_next_entry ) {
 243                         mdb_entry_release( &op, tool_next_entry, 0 );
 244                         tool_next_entry = NULL;
 245                 }
 246
 247                 rc = mdb_tool_entry_get_int( be, id, &tool_next_entry );
 248                 if ( rc == LDAP_NO_SUCH_OBJECT ) {
 249                         goto next;
 250                 }
 251
 252                 assert( tool_next_entry != NULL );
 253
 254                 if ( tool_filter && test_filter( NULL, tool_next_entry, tool_filter ) != LDAP_COMPARE_TRUE )
 255                 {
 256                         mdb_entry_release( &op, tool_next_entry, 0 );
 257                         tool_next_entry = NULL;
 258                         goto next;
 259                 }
 260         }
 261
 262         return id;
 263 }
 264
 265 ID mdb_tool_dn2id_get(
 266         Backend *be,
 267         struct berval *dn
 268 )
 269 {
 270         struct mdb_info *mdb;
 271         Operation op = {0};
 272         Opheader ohdr = {0};
 273         ID id;
 274         int rc;
 275
 276         if ( BER_BVISEMPTY(dn) )
 277                 return 0;
 278
 279         mdb = (struct mdb_info *) be->be_private;
 280
 281         if ( !txn ) {
 282                 rc = mdb_txn_begin( mdb->mi_dbenv, NULL, (slapMode & SLAP_TOOL_READONLY) != 0 ?
 283                         MDB_RDONLY : 0, &txn );
 284                 if ( rc )
 285                         return NOID;
 286         }
 287
 288         op.o_hdr = &ohdr;
 289         op.o_bd = be;
 290         op.o_tmpmemctx = NULL;
 291         op.o_tmpmfuncs = &ch_mfuncs;
 292
 293         rc = mdb_dn2id( &op, txn, dn, &id, NULL, NULL );
 294         if ( rc == MDB_NOTFOUND )
 295                 return NOID;
 296
 297         return id;
 298 }
 299
 300 static int
 301 mdb_tool_entry_get_int( BackendDB *be, ID id, Entry **ep )
 302 {
 303         Operation op = {0};
 304         Opheader ohdr = {0};
 305
 306         Entry *e = NULL;
 307         struct berval dn = BER_BVNULL, ndn = BER_BVNULL;
 308         int rc;
 309
 310         assert( be != NULL );
 311         assert( slapMode & SLAP_TOOL_MODE );
 312
 313         if ( ( tool_filter || tool_base ) && id == previd && tool_next_entry != NULL ) {
 314                 *ep = tool_next_entry;
 315                 tool_next_entry = NULL;
 316                 return LDAP_SUCCESS;
 317         }
 318
 319         if ( id != previd ) {
 320                 key.mv_size = sizeof(ID);
 321                 key.mv_data = &id;
 322                 rc = mdb_cursor_get( cursor, &key, &data, MDB_SET );
 323                 if ( rc ) {
 324                         rc = LDAP_OTHER;
 325                         goto done;
 326                 }
 327         }
 328
 329         op.o_hdr = &ohdr;
 330         op.o_bd = be;
 331         op.o_tmpmemctx = NULL;
 332         op.o_tmpmfuncs = &ch_mfuncs;
 333         if ( slapMode & SLAP_TOOL_READONLY ) {
 334                 rc = mdb_id2name( &op, txn, &idcursor, id, &dn, &ndn );
 335                 if ( rc  ) {
 336                         rc = LDAP_OTHER;
 337                         mdb_entry_return( &op, e );
 338                         e = NULL;
 339                         goto done;
 340                 }
 341                 if ( tool_base != NULL ) {
 342                         if ( !dnIsSuffixScope( &ndn, tool_base, tool_scope ) ) {
 343                                 ch_free( dn.bv_val );
 344                                 ch_free( ndn.bv_val );
 345                                 rc = LDAP_NO_SUCH_OBJECT;
 346                         }
 347                 }
 348         }
 349         rc = mdb_entry_decode( &op, &data, &e );
 350         e->e_id = id;
 351         if ( !BER_BVISNULL( &dn )) {
 352                 e->e_name = dn;
 353                 e->e_nname = ndn;
 354         } else {
 355                 e->e_name.bv_val = NULL;
 356                 e->e_nname.bv_val = NULL;
 357         }
 358
 359 done:
 360         if ( e != NULL ) {
 361                 *ep = e;
 362         }
 363
 364         return rc;
 365 }
 366
 367 Entry*
 368 mdb_tool_entry_get( BackendDB *be, ID id )
 369 {
 370         Entry *e = NULL;
 371
 372         (void)mdb_tool_entry_get_int( be, id, &e );
 373         return e;
 374 }
 375
 376 static int mdb_tool_next_id(
 377         Operation *op,
 378         MDB_txn *tid,
 379         Entry *e,
 380         struct berval *text,
 381         int hole )
 382 {
 383         struct mdb_info *mdb = (struct mdb_info *) op->o_bd->be_private;
 384         struct berval dn = e->e_name;
 385         struct berval ndn = e->e_nname;
 386         struct berval pdn, npdn, nmatched;
 387         ID id, pid = 0;
 388         int rc;
 389
 390         if (ndn.bv_len == 0) {
 391                 e->e_id = 0;
 392                 return 0;
 393         }
 394
 395         rc = mdb_dn2id( op, tid, &ndn, &id, NULL, &nmatched );
 396         if ( rc == MDB_NOTFOUND ) {
 397                 if ( !be_issuffix( op->o_bd, &ndn ) ) {
 398                         ID eid = e->e_id;
 399                         dnParent( &ndn, &npdn );
 400                         if ( nmatched.bv_len != npdn.bv_len ) {
 401                                 dnParent( &dn, &pdn );
 402                                 e->e_name = pdn;
 403                                 e->e_nname = npdn;
 404                                 rc = mdb_tool_next_id( op, tid, e, text, 1 );
 405                                 e->e_name = dn;
 406                                 e->e_nname = ndn;
 407                                 if ( rc ) {
 408                                         return rc;
 409                                 }
 410                                 /* If parent didn't exist, it was created just now
 411                                  * and its ID is now in e->e_id. Make sure the current
 412                                  * entry gets added under the new parent ID.
 413                                  */
 414                                 if ( eid != e->e_id ) {
 415                                         pid = e->e_id;
 416                                 }
 417                         } else {
 418                                 pid = id;
 419                         }
 420                 }
 421                 rc = mdb_next_id( op->o_bd, tid, &e->e_id );
 422                 if ( rc ) {
 423                         snprintf( text->bv_val, text->bv_len,
 424                                 "next_id failed: %s (%d)",
 425                                 mdb_strerror(rc), rc );
 426                 Debug( LDAP_DEBUG_ANY,
 427                         "=> mdb_tool_next_id: %s\n", text->bv_val, 0, 0 );
 428                         return rc;
 429                 }
 430                 rc = mdb_dn2id_add( op, tid, pid, e );
 431                 if ( rc ) {
 432                         snprintf( text->bv_val, text->bv_len,
 433                                 "dn2id_add failed: %s (%d)",
 434                                 mdb_strerror(rc), rc );
 435                         Debug( LDAP_DEBUG_ANY,
 436                                 "=> mdb_tool_next_id: %s\n", text->bv_val, 0, 0 );
 437                 } else if ( hole ) {
 438                         MDB_val key, data;
 439                         if ( nholes == nhmax - 1 ) {
 440                                 if ( holes == hbuf ) {
 441                                         holes = ch_malloc( nhmax * sizeof(dn_id) * 2 );
 442                                         AC_MEMCPY( holes, hbuf, sizeof(hbuf) );
 443                                 } else {
 444                                         holes = ch_realloc( holes, nhmax * sizeof(dn_id) * 2 );
 445                                 }
 446                                 nhmax *= 2;
 447                         }
 448                         ber_dupbv( &holes[nholes].dn, &ndn );
 449                         holes[nholes++].id = e->e_id;
 450                         key.mv_size = sizeof(ID);
 451                         key.mv_data = &e->e_id;
 452                         data.mv_size = 0;
 453                         data.mv_data = NULL;
 454                         rc = mdb_put( tid, mdb->mi_id2entry, &key, &data, MDB_NOOVERWRITE );
 455                         if ( rc == MDB_KEYEXIST )
 456                                 rc = 0;
 457                         if ( rc ) {
 458                                 snprintf( text->bv_val, text->bv_len,
 459                                         "dummy id2entry add failed: %s (%d)",
 460                                         mdb_strerror(rc), rc );
 461                                 Debug( LDAP_DEBUG_ANY,
 462                                         "=> mdb_tool_next_id: %s\n", text->bv_val, 0, 0 );
 463                         }
 464                 }
 465         } else if ( !hole ) {
 466                 unsigned i, j;
 467
 468                 e->e_id = id;
 469
 470                 for ( i=0; i<nholes; i++) {
 471                         if ( holes[i].id == e->e_id ) {
 472                                 free(holes[i].dn.bv_val);
 473                                 for (j=i;j<nholes;j++) holes[j] = holes[j+1];
 474                                 holes[j].id = 0;
 475                                 nholes--;
 476                                 break;
 477                         } else if ( holes[i].id > e->e_id ) {
 478                                 break;
 479                         }
 480                 }
 481         }
 482         return rc;
 483 }
 484
 485 static int
 486 mdb_tool_index_add(
 487         Operation *op,
 488         MDB_txn *txn,
 489         Entry *e )
 490 {
 491         struct mdb_info *mdb = (struct mdb_info *) op->o_bd->be_private;
 492
 493         if ( !mdb->mi_nattrs )
 494                 return 0;
 495
 496 #if 0
 497         if ( slapMode & SLAP_TOOL_QUICK ) {
 498                 IndexRec *ir;
 499                 int i, rc;
 500                 Attribute *a;
 501
 502                 ir = mdb_tool_index_rec;
 503                 memset(ir, 0, mdb->bi_nattrs * sizeof( IndexRec ));
 504
 505                 for ( a = e->e_attrs; a != NULL; a = a->a_next ) {
 506                         rc = mdb_index_recset( mdb, a, a->a_desc->ad_type,
 507                                 &a->a_desc->ad_tags, ir );
 508                         if ( rc )
 509                                 return rc;
 510                 }
 511                 mdb_tool_ix_id = e->e_id;
 512                 mdb_tool_ix_op = op;
 513                 ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex );
 514                 /* Wait for all threads to be ready */
 515                 while ( mdb_tool_index_tcount ) {
 516                         ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main,
 517                                 &mdb_tool_index_mutex );
 518                 }
 519                 for ( i=1; i<slap_tool_thread_max; i++ )
 520                         mdb_tool_index_threads[i] = LDAP_BUSY;
 521                 mdb_tool_index_tcount = slap_tool_thread_max - 1;
 522                 ldap_pvt_thread_cond_broadcast( &mdb_tool_index_cond_work );
 523                 ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex );
 524                 rc = mdb_index_recrun( op, mdb, ir, e->e_id, 0 );
 525                 if ( rc )
 526                         return rc;
 527                 ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex );
 528                 for ( i=1; i<slap_tool_thread_max; i++ ) {
 529                         if ( mdb_tool_index_threads[i] == LDAP_BUSY ) {
 530                                 ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main,
 531                                         &mdb_tool_index_mutex );
 532                                 i--;
 533                                 continue;
 534                         }
 535                         if ( mdb_tool_index_threads[i] ) {
 536                                 rc = mdb_tool_index_threads[i];
 537                                 break;
 538                         }
 539                 }
 540                 ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex );
 541                 return rc;
 542         } else
 543 #endif
 544         {
 545                 return mdb_index_entry_add( op, txn, e );
 546         }
 547 }
 548
 549 ID mdb_tool_entry_put(
 550         BackendDB *be,
 551         Entry *e,
 552         struct berval *text )
 553 {
 554         int rc;
 555         struct mdb_info *mdb;
 556         Operation op = {0};
 557         Opheader ohdr = {0};
 558
 559         assert( be != NULL );
 560         assert( slapMode & SLAP_TOOL_MODE );
 561
 562         assert( text != NULL );
 563         assert( text->bv_val != NULL );
 564         assert( text->bv_val[0] == '\0' );      /* overconservative? */
 565
 566         Debug( LDAP_DEBUG_TRACE, "=> " LDAP_XSTRING(mdb_tool_entry_put)
 567                 "( %ld, \"%s\" )\n", (long) e->e_id, e->e_dn, 0 );
 568
 569         mdb = (struct mdb_info *) be->be_private;
 570
 571         if ( !txn ) {
 572         rc = mdb_txn_begin( mdb->mi_dbenv, NULL, 0, &txn );
 573         if( rc != 0 ) {
 574                 snprintf( text->bv_val, text->bv_len,
 575                         "txn_begin failed: %s (%d)",
 576                         mdb_strerror(rc), rc );
 577                 Debug( LDAP_DEBUG_ANY,
 578                         "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
 579                          text->bv_val, 0, 0 );
 580                 return NOID;
 581         }
 582         }
 583
 584         op.o_hdr = &ohdr;
 585         op.o_bd = be;
 586         op.o_tmpmemctx = NULL;
 587         op.o_tmpmfuncs = &ch_mfuncs;
 588
 589         /* add dn2id indices */
 590         rc = mdb_tool_next_id( &op, txn, e, text, 0 );
 591         if( rc != 0 ) {
 592                 goto done;
 593         }
 594
 595         rc = mdb_tool_index_add( &op, txn, e );
 596         if( rc != 0 ) {
 597                 snprintf( text->bv_val, text->bv_len,
 598                                 "index_entry_add failed: err=%d", rc );
 599                 Debug( LDAP_DEBUG_ANY,
 600                         "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
 601                         text->bv_val, 0, 0 );
 602                 goto done;
 603         }
 604
 605
 606         /* id2entry index */
 607         rc = mdb_id2entry_add( &op, txn, e );
 608         if( rc != 0 ) {
 609                 snprintf( text->bv_val, text->bv_len,
 610                                 "id2entry_add failed: err=%d", rc );
 611                 Debug( LDAP_DEBUG_ANY,
 612                         "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
 613                         text->bv_val, 0, 0 );
 614                 goto done;
 615         }
 616
 617 done:
 618         if( rc == 0 ) {
 619                 mdb_writes++;
 620                 if ( mdb_writes >= mdb_writes_per_commit ) {
 621                         mdb_tool_idl_flush( be, txn );
 622                         rc = mdb_txn_commit( txn );
 623                         mdb_writes = 0;
 624                         txn = NULL;
 625                         if( rc != 0 ) {
 626                                 snprintf( text->bv_val, text->bv_len,
 627                                                 "txn_commit failed: %s (%d)",
 628                                                 mdb_strerror(rc), rc );
 629                                 Debug( LDAP_DEBUG_ANY,
 630                                         "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
 631                                         text->bv_val, 0, 0 );
 632                                 e->e_id = NOID;
 633                         }
 634                 }
 635
 636         } else {
 637                 mdb_txn_abort( txn );
 638                 txn = NULL;
 639                 snprintf( text->bv_val, text->bv_len,
 640                         "txn_aborted! %s (%d)",
 641                         rc == LDAP_OTHER ? "Internal error" :
 642                         mdb_strerror(rc), rc );
 643                 Debug( LDAP_DEBUG_ANY,
 644                         "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
 645                         text->bv_val, 0, 0 );
 646                 e->e_id = NOID;
 647         }
 648
 649         return e->e_id;
 650 }
 651
 652 int mdb_tool_entry_reindex(
 653         BackendDB *be,
 654         ID id,
 655         AttributeDescription **adv )
 656 {
 657         struct mdb_info *mi = (struct mdb_info *) be->be_private;
 658         int rc;
 659         Entry *e;
 660         Operation op = {0};
 661         Opheader ohdr = {0};
 662
 663         Debug( LDAP_DEBUG_ARGS,
 664                 "=> " LDAP_XSTRING(mdb_tool_entry_reindex) "( %ld )\n",
 665                 (long) id, 0, 0 );
 666         assert( tool_base == NULL );
 667         assert( tool_filter == NULL );
 668
 669         /* No indexes configured, nothing to do. Could return an
 670          * error here to shortcut things.
 671          */
 672         if (!mi->mi_attrs) {
 673                 return 0;
 674         }
 675
 676         /* Check for explicit list of attrs to index */
 677         if ( adv ) {
 678                 int i, j, n;
 679
 680                 if ( mi->mi_attrs[0]->ai_desc != adv[0] ) {
 681                         /* count */
 682                         for ( n = 0; adv[n]; n++ ) ;
 683
 684                         /* insertion sort */
 685                         for ( i = 0; i < n; i++ ) {
 686                                 AttributeDescription *ad = adv[i];
 687                                 for ( j = i-1; j>=0; j--) {
 688                                         if ( SLAP_PTRCMP( adv[j], ad ) <= 0 ) break;
 689                                         adv[j+1] = adv[j];
 690                                 }
 691                                 adv[j+1] = ad;
 692                         }
 693                 }
 694
 695                 for ( i = 0; adv[i]; i++ ) {
 696                         if ( mi->mi_attrs[i]->ai_desc != adv[i] ) {
 697                                 for ( j = i+1; j < mi->mi_nattrs; j++ ) {
 698                                         if ( mi->mi_attrs[j]->ai_desc == adv[i] ) {
 699                                                 AttrInfo *ai = mi->mi_attrs[i];
 700                                                 mi->mi_attrs[i] = mi->mi_attrs[j];
 701                                                 mi->mi_attrs[j] = ai;
 702                                                 break;
 703                                         }
 704                                 }
 705                                 if ( j == mi->mi_nattrs ) {
 706                                         Debug( LDAP_DEBUG_ANY,
 707                                                 LDAP_XSTRING(mdb_tool_entry_reindex)
 708                                                 ": no index configured for %s\n",
 709                                                 adv[i]->ad_cname.bv_val, 0, 0 );
 710                                         return -1;
 711                                 }
 712                         }
 713                 }
 714                 mi->mi_nattrs = i;
 715         }
 716
 717         if ( slapMode & SLAP_TRUNCATE_MODE ) {
 718                 int i;
 719                 for ( i=0; i < mi->mi_nattrs; i++ ) {
 720                         rc = mdb_drop( txn, mi->mi_attrs[i]->ai_dbi, 0 );
 721                         if ( rc ) {
 722                                 Debug( LDAP_DEBUG_ANY,
 723                                         LDAP_XSTRING(mdb_tool_entry_reindex)
 724                                         ": (Truncate) mdb_drop(%s) failed: %s (%d)\n",
 725                                         mi->mi_attrs[i]->ai_desc->ad_type->sat_cname.bv_val,
 726                                         mdb_strerror(rc), rc );
 727                                 return -1;
 728                         }
 729                 }
 730                 slapMode ^= SLAP_TRUNCATE_MODE;
 731         }
 732
 733         e = mdb_tool_entry_get( be, id );
 734
 735         if( e == NULL ) {
 736                 Debug( LDAP_DEBUG_ANY,
 737                         LDAP_XSTRING(mdb_tool_entry_reindex)
 738                         ": could not locate id=%ld\n",
 739                         (long) id, 0, 0 );
 740                 return -1;
 741         }
 742
 743         if ( !txi ) {
 744                 rc = mdb_txn_begin( mi->mi_dbenv, NULL, 0, &txi );
 745                 if( rc != 0 ) {
 746                         Debug( LDAP_DEBUG_ANY,
 747                                 "=> " LDAP_XSTRING(mdb_tool_entry_reindex) ": "
 748                                 "txn_begin failed: %s (%d)\n",
 749                                 mdb_strerror(rc), rc, 0 );
 750                         goto done;
 751                 }
 752         }
 753
 754         /*
 755          * just (re)add them for now
 756          * assume that some other routine (not yet implemented)
 757          * will zap index databases
 758          *
 759          */
 760
 761         Debug( LDAP_DEBUG_TRACE,
 762                 "=> " LDAP_XSTRING(mdb_tool_entry_reindex) "( %ld, \"%s\" )\n",
 763                 (long) id, e->e_dn, 0 );
 764
 765         op.o_hdr = &ohdr;
 766         op.o_bd = be;
 767         op.o_tmpmemctx = NULL;
 768         op.o_tmpmfuncs = &ch_mfuncs;
 769
 770         rc = mdb_tool_index_add( &op, txi, e );
 771
 772 done:
 773         if( rc == 0 ) {
 774                 mdb_writes++;
 775                 if ( mdb_writes >= mdb_writes_per_commit ) {
 776                         mdb_tool_idl_flush( be, txi );
 777                         rc = mdb_txn_commit( txi );
 778                         if( rc != 0 ) {
 779                                 Debug( LDAP_DEBUG_ANY,
 780                                         "=> " LDAP_XSTRING(mdb_tool_entry_reindex)
 781                                         ": txn_commit failed: %s (%d)\n",
 782                                         mdb_strerror(rc), rc, 0 );
 783                                 e->e_id = NOID;
 784                         }
 785                         txi = NULL;
 786                 }
 787
 788         } else {
 789                 mdb_txn_abort( txi );
 790                 Debug( LDAP_DEBUG_ANY,
 791                         "=> " LDAP_XSTRING(mdb_tool_entry_reindex)
 792                         ": txn_aborted! err=%d\n",
 793                         rc, 0, 0 );
 794                 e->e_id = NOID;
 795                 txi = NULL;
 796         }
 797         mdb_entry_release( &op, e, 0 );
 798
 799         return rc;
 800 }
 801
 802 ID mdb_tool_entry_modify(
 803         BackendDB *be,
 804         Entry *e,
 805         struct berval *text )
 806 {
 807         int rc;
 808         struct mdb_info *mdb;
 809         MDB_txn *tid;
 810         Operation op = {0};
 811         Opheader ohdr = {0};
 812
 813         assert( be != NULL );
 814         assert( slapMode & SLAP_TOOL_MODE );
 815
 816         assert( text != NULL );
 817         assert( text->bv_val != NULL );
 818         assert( text->bv_val[0] == '\0' );      /* overconservative? */
 819
 820         assert ( e->e_id != NOID );
 821
 822         Debug( LDAP_DEBUG_TRACE,
 823                 "=> " LDAP_XSTRING(mdb_tool_entry_modify) "( %ld, \"%s\" )\n",
 824                 (long) e->e_id, e->e_dn, 0 );
 825
 826         mdb = (struct mdb_info *) be->be_private;
 827
 828         if( cursor ) {
 829                 mdb_cursor_close( cursor );
 830                 cursor = NULL;
 831         }
 832         rc = mdb_txn_begin( mdb->mi_dbenv, NULL, 0, &tid );
 833         if( rc != 0 ) {
 834                 snprintf( text->bv_val, text->bv_len,
 835                         "txn_begin failed: %s (%d)",
 836                         mdb_strerror(rc), rc );
 837                 Debug( LDAP_DEBUG_ANY,
 838                         "=> " LDAP_XSTRING(mdb_tool_entry_modify) ": %s\n",
 839                          text->bv_val, 0, 0 );
 840                 return NOID;
 841         }
 842
 843         op.o_hdr = &ohdr;
 844         op.o_bd = be;
 845         op.o_tmpmemctx = NULL;
 846         op.o_tmpmfuncs = &ch_mfuncs;
 847
 848         /* id2entry index */
 849         rc = mdb_id2entry_update( &op, tid, e );
 850         if( rc != 0 ) {
 851                 snprintf( text->bv_val, text->bv_len,
 852                                 "id2entry_update failed: err=%d", rc );
 853                 Debug( LDAP_DEBUG_ANY,
 854                         "=> " LDAP_XSTRING(mdb_tool_entry_modify) ": %s\n",
 855                         text->bv_val, 0, 0 );
 856                 goto done;
 857         }
 858
 859 done:
 860         if( rc == 0 ) {
 861                 rc = mdb_txn_commit( tid );
 862                 if( rc != 0 ) {
 863                         snprintf( text->bv_val, text->bv_len,
 864                                         "txn_commit failed: %s (%d)",
 865                                         mdb_strerror(rc), rc );
 866                         Debug( LDAP_DEBUG_ANY,
 867                                 "=> " LDAP_XSTRING(mdb_tool_entry_modify) ": "
 868                                 "%s\n", text->bv_val, 0, 0 );
 869                         e->e_id = NOID;
 870                 }
 871
 872         } else {
 873                 mdb_txn_abort( tid );
 874                 snprintf( text->bv_val, text->bv_len,
 875                         "txn_aborted! %s (%d)",
 876                         mdb_strerror(rc), rc );
 877                 Debug( LDAP_DEBUG_ANY,
 878                         "=> " LDAP_XSTRING(mdb_tool_entry_modify) ": %s\n",
 879                         text->bv_val, 0, 0 );
 880                 e->e_id = NOID;
 881         }
 882
 883         return e->e_id;
 884 }
 885
 886 #if 0
 887 static void *
 888 mdb_tool_index_task( void *ctx, void *ptr )
 889 {
 890         int base = *(int *)ptr;
 891
 892         free( ptr );
 893         while ( 1 ) {
 894                 ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex );
 895                 mdb_tool_index_tcount--;
 896                 if ( !mdb_tool_index_tcount )
 897                         ldap_pvt_thread_cond_signal( &mdb_tool_index_cond_main );
 898                 ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_work,
 899                         &mdb_tool_index_mutex );
 900                 if ( slapd_shutdown ) {
 901                         mdb_tool_index_tcount--;
 902                         if ( !mdb_tool_index_tcount )
 903                                 ldap_pvt_thread_cond_signal( &mdb_tool_index_cond_main );
 904                         ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex );
 905                         break;
 906                 }
 907                 ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex );
 908
 909                 mdb_tool_index_threads[base] = mdb_index_recrun( mdb_tool_ix_op,
 910                         mdb_tool_info, mdb_tool_index_rec, mdb_tool_ix_id, base );
 911         }
 912
 913         return NULL;
 914 }
 915 #endif
 916
 917 static int
 918 mdb_tool_idl_cmp( const void *v1, const void *v2 )
 919 {
 920         const mdb_tool_idl_cache *c1 = v1, *c2 = v2;
 921         int rc;
 922
 923         if (( rc = c1->kstr.bv_len - c2->kstr.bv_len )) return rc;
 924         return memcmp( c1->kstr.bv_val, c2->kstr.bv_val, c1->kstr.bv_len );
 925 }
 926
 927 static int
 928 mdb_tool_idl_flush_one( MDB_cursor *mc, mdb_tool_idl_cache *ic )
 929 {
 930         mdb_tool_idl_cache_entry *ice;
 931         MDB_val key, data[2];
 932         int i, rc;
 933         ID id, nid;
 934
 935         /* Freshly allocated, ignore it */
 936         if ( !ic->head && ic->count <= MDB_IDL_DB_SIZE ) {
 937                 return 0;
 938         }
 939
 940         key.mv_data = ic->kstr.bv_val;
 941         key.mv_size = ic->kstr.bv_len;
 942
 943         if ( ic->count > MDB_IDL_DB_SIZE ) {
 944                 while ( ic->flags & WAS_FOUND ) {
 945                         rc = mdb_cursor_get( mc, &key, data, MDB_SET );
 946                         if ( rc ) {
 947                                 /* FIXME: find out why this happens */
 948                                 ic->flags = 0;
 949                                 break;
 950                         }
 951                         if ( ic->flags & WAS_RANGE ) {
 952                                 /* Skip lo */
 953                                 rc = mdb_cursor_get( mc, &key, data, MDB_NEXT_DUP );
 954
 955                                 /* Get hi */
 956                                 rc = mdb_cursor_get( mc, &key, data, MDB_NEXT_DUP );
 957
 958                                 /* Store range hi */
 959                                 data[0].mv_data = &ic->last;
 960                                 rc = mdb_cursor_put( mc, &key, data, MDB_CURRENT );
 961                         } else {
 962                                 /* Delete old data, replace with range */
 963                                 ic->first = *(ID *)data[0].mv_data;
 964                                 mdb_cursor_del( mc, MDB_NODUPDATA );
 965                         }
 966                         break;
 967                 }
 968                 if ( !(ic->flags & WAS_RANGE)) {
 969                         /* range, didn't exist before */
 970                         nid = 0;
 971                         data[0].mv_size = sizeof(ID);
 972                         data[0].mv_data = &nid;
 973                         rc = mdb_cursor_put( mc, &key, data, 0 );
 974                         if ( rc == 0 ) {
 975                                 data[0].mv_data = &ic->first;
 976                                 rc = mdb_cursor_put( mc, &key, data, 0 );
 977                                 if ( rc == 0 ) {
 978                                         data[0].mv_data = &ic->last;
 979                                         rc = mdb_cursor_put( mc, &key, data, 0 );
 980                                 }
 981                         }
 982                         if ( rc ) {
 983                                 rc = -1;
 984                         }
 985                 }
 986         } else {
 987                 /* Normal write */
 988                 int n;
 989
 990                 data[0].mv_size = sizeof(ID);
 991                 rc = 0;
 992                 i = ic->offset;
 993                 for ( ice = ic->head, n=0; ice; ice = ice->next, n++ ) {
 994                         int end;
 995                         if ( ice->next ) {
 996                                 end = IDBLOCK;
 997                         } else {
 998                                 end = ic->count & (IDBLOCK-1);
 999                                 if ( !end )
1000                                         end = IDBLOCK;
1001                         }
1002                         data[1].mv_size = end - i;
1003                         data[0].mv_data = &ice->ids[i];
1004                         i = 0;
1005                         rc = mdb_cursor_put( mc, &key, data, MDB_NODUPDATA|MDB_APPEND|MDB_MULTIPLE );
1006                         if ( rc ) {
1007                                 if ( rc == MDB_KEYEXIST ) {
1008                                         rc = 0;
1009                                         continue;
1010                                 }
1011                                 rc = -1;
1012                                 break;
1013                         }
1014                 }
1015                 if ( ic->head ) {
1016                         ic->tail->next = mdb_tool_idl_free_list;
1017                         mdb_tool_idl_free_list = ic->head;
1018                 }
1019         }
1020         ch_free( ic );
1021         return rc;
1022 }
1023
1024 static int
1025 mdb_tool_idl_flush_db( MDB_txn *txn, MDB_dbi dbi, Avlnode *root )
1026 {
1027         MDB_cursor *mc;
1028         int rc;
1029
1030         mdb_cursor_open( txn, dbi, &mc );
1031         root = tavl_end( root, TAVL_DIR_LEFT );
1032         do {
1033                 rc = mdb_tool_idl_flush_one( mc, root->avl_data );
1034                 if ( rc != -1 )
1035                         rc = 0;
1036         } while ((root = tavl_next(root, TAVL_DIR_RIGHT)));
1037         mdb_cursor_close( mc );
1038
1039         return rc;
1040 }
1041
1042 static int
1043 mdb_tool_idl_flush( BackendDB *be, MDB_txn *txn )
1044 {
1045         struct mdb_info *mdb = (struct mdb_info *) be->be_private;
1046         int rc = 0;
1047         unsigned int i;
1048
1049         for ( i=MDB_NDB; i < mdb->mi_nattrs+MDB_NDB; i++ ) {
1050                 if ( !mdb_tool_roots[i] ) continue;
1051                 rc = mdb_tool_idl_flush_db( txn, i, mdb_tool_roots[i] );
1052                 tavl_free(mdb_tool_roots[i], NULL);
1053                 mdb_tool_roots[i] = NULL;
1054                 if ( rc )
1055                         break;
1056         }
1057         return rc;
1058 }
1059
1060 int mdb_tool_idl_add(
1061         MDB_cursor *mc,
1062         struct berval *keys,
1063         ID id )
1064 {
1065         MDB_dbi dbi;
1066         mdb_tool_idl_cache *ic, itmp;
1067         mdb_tool_idl_cache_entry *ice;
1068         int i, rc, lcount;
1069
1070         dbi = mdb_cursor_dbi(mc);
1071         for (i=0; keys[i].bv_val; i++) {
1072         itmp.kstr = keys[i];
1073         ic = tavl_find( (Avlnode *)mdb_tool_roots[dbi], &itmp, mdb_tool_idl_cmp );
1074
1075         /* No entry yet, create one */
1076         if ( !ic ) {
1077                 MDB_val key, data;
1078                 ID nid;
1079                 int rc;
1080
1081                 ic = ch_malloc( sizeof( mdb_tool_idl_cache ) + itmp.kstr.bv_len );
1082                 ic->kstr.bv_len = itmp.kstr.bv_len;
1083                 ic->kstr.bv_val = (char *)(ic+1);
1084                 memcpy( ic->kstr.bv_val, itmp.kstr.bv_val, ic->kstr.bv_len );
1085                 ic->head = ic->tail = NULL;
1086                 ic->last = 0;
1087                 ic->count = 0;
1088                 ic->offset = 0;
1089                 ic->flags = 0;
1090                 tavl_insert( (Avlnode **)&mdb_tool_roots[dbi], ic, mdb_tool_idl_cmp,
1091                         avl_dup_error );
1092
1093                 /* load existing key count here */
1094                 key.mv_size = keys[i].bv_len;
1095                 key.mv_data = keys[i].bv_val;
1096                 rc = mdb_cursor_get( mc, &key, &data, MDB_SET );
1097                 if ( rc == 0 ) {
1098                         ic->flags |= WAS_FOUND;
1099                         nid = *(ID *)data.mv_data;
1100                         if ( nid == 0 ) {
1101                                 ic->count = MDB_IDL_DB_SIZE+1;
1102                                 ic->flags |= WAS_RANGE;
1103                         } else {
1104                                 size_t count;
1105
1106                                 mdb_cursor_count( mc, &count );
1107                                 ic->count = count;
1108                                 ic->first = nid;
1109                                 ic->offset = count & (IDBLOCK-1);
1110                         }
1111                 }
1112         }
1113         /* are we a range already? */
1114         if ( ic->count > MDB_IDL_DB_SIZE ) {
1115                 ic->last = id;
1116                 continue;
1117         /* Are we at the limit, and converting to a range? */
1118         } else if ( ic->count == MDB_IDL_DB_SIZE ) {
1119                 int n;
1120                 for ( ice = ic->head, n=0; ice; ice = ice->next, n++ )
1121                         /* counting */ ;
1122                 if ( n ) {
1123                         ic->tail->next = mdb_tool_idl_free_list;
1124                         mdb_tool_idl_free_list = ic->head;
1125                 }
1126                 ic->head = ic->tail = NULL;
1127                 ic->last = id;
1128                 ic->count++;
1129                 continue;
1130         }
1131         /* No free block, create that too */
1132         lcount = ic->count & (IDBLOCK-1);
1133         if ( !ic->tail || lcount == 0) {
1134                 ice = NULL;
1135                 if ( mdb_tool_idl_free_list ) {
1136                         ice = mdb_tool_idl_free_list;
1137                         mdb_tool_idl_free_list = ice->next;
1138                 }
1139                 if ( !ice ) {
1140                         ice = ch_malloc( sizeof( mdb_tool_idl_cache_entry ));
1141                 }
1142                 ice->next = NULL;
1143                 if ( !ic->head ) {
1144                         ic->head = ice;
1145                 } else {
1146                         ic->tail->next = ice;
1147                 }
1148                 ic->tail = ice;
1149                 if ( !ic->count )
1150                         ic->first = id;
1151         }
1152         ice = ic->tail;
1153         if (!lcount || ice->ids[lcount-1] != id)
1154                 ice->ids[lcount] = id;
1155         ic->count++;
1156         }
1157
1158         return 0;
1159 }