git.sur5r.net Git - openldap/blob - servers/slapd/back-ldbm/idl.c

   1 /* idl.c - ldap id list handling routines */
   2 /* $OpenLDAP$ */
   3 /*
   4  * Copyright 1998-2002 The OpenLDAP Foundation, All Rights Reserved.
   5  * COPYING RESTRICTIONS APPLY, see COPYRIGHT file
   6  */
   7
   8 #include "portable.h"
   9
  10 #include <stdio.h>
  11
  12 #include <ac/string.h>
  13 #include <ac/socket.h>
  14
  15 #include "slap.h"
  16 #include "back-ldbm.h"
  17
  18 static ID_BLOCK* idl_dup( ID_BLOCK *idl );
  19
  20 static void cont_alloc( Datum *cont, Datum *key )
  21 {
  22         ldbm_datum_init( *cont );
  23         cont->dsize = 1 + sizeof(ID) + key->dsize;
  24         cont->dptr = ch_malloc( cont->dsize );
  25
  26         * (unsigned char *) cont->dptr = SLAP_INDEX_CONT_PREFIX;
  27
  28         AC_MEMCPY( &((unsigned char *)cont->dptr)[1 + sizeof(ID)],
  29                 key->dptr, key->dsize );
  30 }
  31
  32 static void cont_id( Datum *cont, ID id )
  33 {
  34         unsigned int i;
  35
  36         for( i=1; i <= sizeof(id); i++) {
  37                 ((unsigned char *)cont->dptr)[i] = (unsigned char)(id & 0xFF);
  38                 id >>= 8;
  39         }
  40
  41 }
  42
  43 static void cont_free( Datum *cont )
  44 {
  45         ch_free( cont->dptr );
  46 }
  47
  48 #ifdef LDBM_DEBUG_IDL
  49 static void idl_check(ID_BLOCK *idl)
  50 {
  51         int i;
  52         ID_BLOCK last;
  53
  54         if( ID_BLOCK_INDIRECT(idl) || ID_BLOCK_ALLIDS(idl)
  55                 || ID_BLOCK_NIDS(idl) <= 1 )
  56         {
  57                 return;
  58         }
  59
  60         for( last = ID_BLOCK_ID(idl, 0), i = 1;
  61                 i < ID_BLOCK_NIDS(idl);
  62                 last = ID_BLOCK_ID(idl, i), i++ )
  63         {
  64                 assert (last < ID_BLOCK_ID(idl, i) );
  65         }
  66 }
  67 #endif
  68
  69 /* Allocate an ID_BLOCK with room for nids ids */
  70 ID_BLOCK *
  71 idl_alloc( unsigned int nids )
  72 {
  73         ID_BLOCK        *new;
  74
  75         /* nmax + nids + space for the ids */
  76         new = (ID_BLOCK *) ch_calloc( (ID_BLOCK_IDS_OFFSET + nids), sizeof(ID) );
  77         ID_BLOCK_NMAX(new) = nids;
  78         ID_BLOCK_NIDS(new) = 0;
  79
  80         return( new );
  81 }
  82
  83
  84 /* Allocate an empty ALLIDS ID_BLOCK */
  85 ID_BLOCK        *
  86 idl_allids( Backend *be )
  87 {
  88         ID_BLOCK        *idl;
  89         ID              id;
  90
  91         idl = idl_alloc( 0 );
  92         ID_BLOCK_NMAX(idl) = ID_BLOCK_ALLIDS_VALUE;
  93         if ( next_id_get( be, &id ) ) {
  94                 return NULL;
  95         }
  96         ID_BLOCK_NIDS(idl) = id;
  97
  98         return( idl );
  99 }
 100
 101 /* Free an ID_BLOCK */
 102 void
 103 idl_free( ID_BLOCK *idl )
 104 {
 105         if ( idl == NULL ) {
 106 #ifdef NEW_LOGGING
 107                 LDAP_LOG( INDEX, INFO, "idl_free: called with NULL pointer\n" , 0,0,0);
 108 #else
 109                 Debug( LDAP_DEBUG_TRACE,
 110                         "idl_free: called with NULL pointer\n",
 111                         0, 0, 0 );
 112 #endif
 113
 114                 return;
 115         }
 116
 117         free( (char *) idl );
 118 }
 119
 120
 121 /* Fetch an single ID_BLOCK from the cache */
 122 static ID_BLOCK *
 123 idl_fetch_one(
 124     Backend             *be,
 125     DBCache     *db,
 126     Datum               key
 127 )
 128 {
 129         Datum   data;
 130         ID_BLOCK        *idl;
 131
 132         /* Debug( LDAP_DEBUG_TRACE, "=> idl_fetch_one\n", 0, 0, 0 ); */
 133
 134         data = ldbm_cache_fetch( db, key );
 135
 136         if( data.dptr == NULL ) {
 137                 return NULL;
 138         }
 139
 140         idl = (ID_BLOCK *) data.dptr;
 141         if ( ID_BLOCK_ALLIDS(idl) ) {
 142                 /* make sure we have the current value of highest id */
 143                 idl = idl_allids( be );
 144         } else {
 145                 idl = idl_dup((ID_BLOCK *) data.dptr);
 146         }
 147
 148         ldbm_datum_free( db->dbc_db, data );
 149
 150         return idl;
 151 }
 152
 153
 154 /* Fetch a set of ID_BLOCKs from the cache
 155  *      if not INDIRECT
 156  *              if block return is an ALLIDS block,
 157  *                      return an new ALLIDS block
 158  *              otherwise
 159  *                      return block
 160  *      construct super block from all blocks referenced by INDIRECT block
 161  *      return super block
 162  */
 163 ID_BLOCK *
 164 idl_fetch(
 165     Backend             *be,
 166     DBCache     *db,
 167     Datum               key
 168 )
 169 {
 170         Datum   data;
 171         ID_BLOCK        *idl;
 172         ID_BLOCK        **tmp;
 173         int     nids;
 174         unsigned i;
 175
 176         idl = idl_fetch_one( be, db, key );
 177
 178         if ( idl == NULL ) {
 179                 return NULL;
 180         }
 181
 182         if ( ID_BLOCK_ALLIDS(idl) ) {
 183                 /* all ids block */
 184                 return( idl );
 185         }
 186
 187         if ( ! ID_BLOCK_INDIRECT( idl ) ) {
 188                 /* regular block */
 189                 return( idl );
 190         }
 191
 192         /*
 193          * this is an indirect block which points to other blocks.
 194          * we need to read in all the blocks it points to and construct
 195          * a big id list containing all the ids, which we will return.
 196          */
 197
 198 #ifndef USE_INDIRECT_NIDS
 199         /* count the number of blocks & allocate space for pointers to them */
 200         for ( i = 0; !ID_BLOCK_NOID(idl, i); i++ )
 201                 ;       /* NULL */
 202 #else
 203         i = ID_BLOCK_NIDS(idl);
 204 #endif
 205         tmp = (ID_BLOCK **) ch_malloc( (i + 1) * sizeof(ID_BLOCK *) );
 206
 207         /* read in all the blocks */
 208         cont_alloc( &data, &key );
 209         nids = 0;
 210 #ifndef USE_INDIRECT_NIDS
 211         for ( i = 0; !ID_BLOCK_NOID(idl, i); i++ ) {
 212 #else
 213         for ( i = 0; i < ID_BLOCK_NIDS(idl); i++ ) {
 214 #endif
 215                 cont_id( &data, ID_BLOCK_ID(idl, i) );
 216
 217                 if ( (tmp[i] = idl_fetch_one( be, db, data )) == NULL ) {
 218 #ifdef NEW_LOGGING
 219                         LDAP_LOG( INDEX, INFO,
 220                                    "idl_fetch: idl_fetch_one returned NULL\n", 0,0,0 );
 221 #else
 222                         Debug( LDAP_DEBUG_ANY,
 223                             "idl_fetch: one returned NULL\n", 0, 0, 0 );
 224 #endif
 225
 226                         continue;
 227                 }
 228
 229                 nids += ID_BLOCK_NIDS(tmp[i]);
 230         }
 231         tmp[i] = NULL;
 232         cont_free( &data );
 233         idl_free( idl );
 234
 235         /* allocate space for the big block */
 236         idl = idl_alloc( nids );
 237         ID_BLOCK_NIDS(idl) = nids;
 238         nids = 0;
 239
 240         /* copy in all the ids from the component blocks */
 241         for ( i = 0; tmp[i] != NULL; i++ ) {
 242                 if ( tmp[i] == NULL ) {
 243                         continue;
 244                 }
 245
 246                 AC_MEMCPY(
 247                         (char *) &ID_BLOCK_ID(idl, nids),
 248                         (char *) &ID_BLOCK_ID(tmp[i], 0),
 249                         ID_BLOCK_NIDS(tmp[i]) * sizeof(ID) );
 250                 nids += ID_BLOCK_NIDS(tmp[i]);
 251
 252                 idl_free( tmp[i] );
 253         }
 254         free( (char *) tmp );
 255
 256 #ifdef LDBM_DEBUG_IDL
 257         idl_check(idl);
 258 #endif
 259
 260 #ifdef NEW_LOGGING
 261         LDAP_LOG( INDEX, ENTRY,
 262                    "idl_fetch: %ld ids (%ld max)\n",
 263                    ID_BLOCK_NIDS(idl), ID_BLOCK_NMAXN(idl), 0 );
 264 #else
 265         Debug( LDAP_DEBUG_TRACE, "<= idl_fetch %ld ids (%ld max)\n",
 266                ID_BLOCK_NIDS(idl), ID_BLOCK_NMAXN(idl), 0 );
 267 #endif
 268
 269         return( idl );
 270 }
 271
 272
 273 /* store a single block */
 274 static int
 275 idl_store(
 276     Backend             *be,
 277     DBCache     *db,
 278     Datum               key,
 279     ID_BLOCK            *idl
 280 )
 281 {
 282         int     rc, flags;
 283         Datum   data;
 284
 285 #ifdef LDBM_DEBUG_IDL
 286         idl_check(idl);
 287 #endif
 288
 289         ldbm_datum_init( data );
 290
 291         /* Debug( LDAP_DEBUG_TRACE, "=> idl_store\n", 0, 0, 0 ); */
 292
 293         data.dptr = (char *) idl;
 294         data.dsize = (ID_BLOCK_IDS_OFFSET + ID_BLOCK_NMAXN(idl)) * sizeof(ID);
 295
 296         flags = LDBM_REPLACE;
 297         rc = ldbm_cache_store( db, key, data, flags );
 298
 299 #ifdef LDBM_DEBUG
 300         Statslog( LDAP_DEBUG_STATS, "<= idl_store(): rc=%d\n",
 301                 rc, 0, 0, 0, 0 );
 302 #endif
 303
 304         /* Debug( LDAP_DEBUG_TRACE, "<= idl_store %d\n", rc, 0, 0 ); */
 305         return( rc );
 306 }
 307
 308 /* Binary search for id in block, return index
 309  *    an index is always returned, even with no match. If no
 310  * match, the returned index is the insertion point.
 311  */
 312 static unsigned int
 313 idl_find(
 314     ID_BLOCK    *b,
 315     ID          id
 316 )
 317 {
 318         int lo=0, hi=ID_BLOCK_NIDS(b)-1, nr=0;
 319
 320         for (;lo<=hi;)
 321         {
 322             nr = ( lo + hi ) / 2;
 323             if (ID_BLOCK_ID(b, nr) == id)
 324                 break;
 325             if (ID_BLOCK_ID(b, nr) > id)
 326                 hi = nr - 1;
 327             else
 328                 lo = nr + 1;
 329         }
 330         return nr;
 331 }
 332
 333 /* split the block at id
 334  *      locate ID greater than or equal to id.
 335  */
 336 static void
 337 idl_split_block(
 338     ID_BLOCK    *b,
 339     ID          id,
 340     ID_BLOCK    **right,
 341     ID_BLOCK    **left
 342 )
 343 {
 344         unsigned int    nr, nl;
 345
 346         /* find where to split the block */
 347         nr = idl_find(b, id);
 348         if ( ID_BLOCK_ID(b,nr) < id )
 349                 nr++;
 350
 351         nl = ID_BLOCK_NIDS(b) - nr;
 352
 353         *right = idl_alloc( nr == 0 ? 1 : nr );
 354         *left = idl_alloc( nl + (nr == 0 ? 0 : 1));
 355
 356         /*
 357          * everything before the id being inserted in the first block
 358          * unless there is nothing, in which case the id being inserted
 359          * goes there.
 360          */
 361         if ( nr == 0 ) {
 362                 ID_BLOCK_NIDS(*right) = 1;
 363                 ID_BLOCK_ID(*right, 0) = id;
 364         } else {
 365                 AC_MEMCPY(
 366                         (char *) &ID_BLOCK_ID(*right, 0),
 367                         (char *) &ID_BLOCK_ID(b, 0),
 368                         nr * sizeof(ID) );
 369                 ID_BLOCK_NIDS(*right) = nr;
 370                 ID_BLOCK_ID(*left, 0) = id;
 371         }
 372
 373         /* the id being inserted & everything after in the second block */
 374         AC_MEMCPY(
 375                 (char *) &ID_BLOCK_ID(*left, (nr == 0 ? 0 : 1)),
 376             (char *) &ID_BLOCK_ID(b, nr),
 377                 nl * sizeof(ID) );
 378         ID_BLOCK_NIDS(*left) = nl + (nr == 0 ? 0 : 1);
 379
 380 #ifdef LDBM_DEBUG_IDL
 381         idl_check(*right);
 382         idl_check(*left);
 383 #endif
 384 }
 385
 386
 387 /*
 388  * idl_change_first - called when an indirect block's first key has
 389  * changed, meaning it needs to be stored under a new key, and the
 390  * header block pointing to it needs updating.
 391  */
 392 static int
 393 idl_change_first(
 394     Backend             *be,
 395     DBCache     *db,
 396     Datum               hkey,           /* header block key     */
 397     ID_BLOCK            *h,             /* header block         */
 398     int                 pos,            /* pos in h to update   */
 399     Datum               bkey,           /* data block key       */
 400     ID_BLOCK            *b              /* data block           */
 401 )
 402 {
 403         int     rc;
 404
 405         /* Debug( LDAP_DEBUG_TRACE, "=> idl_change_first\n", 0, 0, 0 ); */
 406
 407         /* delete old key block */
 408         if ( (rc = ldbm_cache_delete( db, bkey )) != 0 ) {
 409 #ifdef NEW_LOGGING
 410                 LDAP_LOG( INDEX, INFO,
 411                            "idl_change_first: ldbm_cache_delete returned %d\n", rc, 0, 0 );
 412 #else
 413                 Debug( LDAP_DEBUG_ANY,
 414                     "idl_change_first: ldbm_cache_delete returned %d\n",
 415                         rc, 0, 0 );
 416 #endif
 417
 418                 return( rc );
 419         }
 420
 421         /* write block with new key */
 422         cont_id( &bkey, ID_BLOCK_ID(b, 0) );
 423
 424         if ( (rc = idl_store( be, db, bkey, b )) != 0 ) {
 425 #ifdef NEW_LOGGING
 426                 LDAP_LOG( INDEX, INFO,
 427                            "idl_change_first: idl_store returned %d\n", rc, 0, 0 );
 428 #else
 429                 Debug( LDAP_DEBUG_ANY,
 430                     "idl_change_first: idl_store returned %d\n", rc, 0, 0 );
 431 #endif
 432
 433                 return( rc );
 434         }
 435
 436         /* update + write indirect header block */
 437         ID_BLOCK_ID(h, pos) = ID_BLOCK_ID(b, 0);
 438         if ( (rc = idl_store( be, db, hkey, h )) != 0 ) {
 439 #ifdef NEW_LOGGING
 440                 LDAP_LOG( INDEX, INFO,
 441                            "idl_change_first: idl_store returned %s\n", rc, 0, 0 );
 442 #else
 443                 Debug( LDAP_DEBUG_ANY,
 444                     "idl_change_first: idl_store returned %d\n", rc, 0, 0 );
 445 #endif
 446
 447                 return( rc );
 448         }
 449
 450         return( 0 );
 451 }
 452
 453
 454 int
 455 idl_insert_key(
 456     Backend             *be,
 457     DBCache     *db,
 458     Datum               key,
 459     ID                  id
 460 )
 461 {
 462         int     i, j, first, rc = 0;
 463         ID_BLOCK        *idl, *tmp, *tmp2, *tmp3;
 464         Datum   k2;
 465
 466         if ( (idl = idl_fetch_one( be, db, key )) == NULL ) {
 467                 idl = idl_alloc( 1 );
 468                 ID_BLOCK_ID(idl, ID_BLOCK_NIDS(idl)++) = id;
 469                 rc = idl_store( be, db, key, idl );
 470
 471                 idl_free( idl );
 472                 return( rc );
 473         }
 474
 475         if ( ID_BLOCK_ALLIDS( idl ) ) {
 476                 /* ALLIDS */
 477                 idl_free( idl );
 478                 return 0;
 479         }
 480
 481         if ( ! ID_BLOCK_INDIRECT( idl ) ) {
 482                 /* regular block */
 483                 switch ( idl_insert( &idl, id, db->dbc_maxids ) ) {
 484                 case 0:         /* id inserted - store the updated block */
 485                 case 1:
 486                         rc = idl_store( be, db, key, idl );
 487                         break;
 488
 489                 case 2:         /* id already there - nothing to do */
 490                         rc = 0;
 491                         break;
 492
 493                 case 3:         /* id not inserted - block must be split */
 494                         /* check threshold for marking this an all-id block */
 495                         if ( db->dbc_maxindirect < 2 ) {
 496                                 idl_free( idl );
 497                                 idl = idl_allids( be );
 498                                 rc = idl_store( be, db, key, idl );
 499                                 break;
 500                         }
 501
 502                         idl_split_block( idl, id, &tmp, &tmp2 );
 503                         idl_free( idl );
 504
 505                         /* create the header indirect block */
 506 #ifndef USE_INDIRECT_NIDS
 507                         idl = idl_alloc( 3 );
 508                         ID_BLOCK_NMAX(idl) = 3;
 509                         ID_BLOCK_NIDS(idl) = ID_BLOCK_INDIRECT_VALUE;
 510                         ID_BLOCK_ID(idl, 0) = ID_BLOCK_ID(tmp, 0);
 511                         ID_BLOCK_ID(idl, 1) = ID_BLOCK_ID(tmp2, 0);
 512                         ID_BLOCK_ID(idl, 2) = NOID;
 513 #else
 514                         idl = idl_alloc( 2 );
 515                         ID_BLOCK_NMAX(idl) = 2 | ID_BLOCK_INDIRECT_VALUE;
 516                         ID_BLOCK_NIDS(idl) = 2;
 517                         ID_BLOCK_ID(idl, 0) = ID_BLOCK_ID(tmp, 0);
 518                         ID_BLOCK_ID(idl, 1) = ID_BLOCK_ID(tmp2, 0);
 519 #endif
 520
 521                         /* store it */
 522                         rc = idl_store( be, db, key, idl );
 523
 524                         cont_alloc( &k2, &key );
 525                         cont_id( &k2, ID_BLOCK_ID(tmp, 0) );
 526
 527                         rc = idl_store( be, db, k2, tmp );
 528
 529                         cont_id( &k2, ID_BLOCK_ID(tmp2, 0) );
 530                         rc = idl_store( be, db, k2, tmp2 );
 531
 532                         cont_free( &k2 );
 533
 534                         idl_free( tmp );
 535                         idl_free( tmp2 );
 536                         break;
 537                 }
 538
 539                 idl_free( idl );
 540                 return( rc );
 541         }
 542
 543         /*
 544          * this is an indirect block which points to other blocks.
 545          * we need to read in the block into which the id should be
 546          * inserted, then insert the id and store the block.  we might
 547          * have to split the block if it is full, which means we also
 548          * need to write a new "header" block.
 549          */
 550
 551 #ifndef USE_INDIRECT_NIDS
 552         /* select the block to try inserting into *//* XXX linear search XXX */
 553         for ( i = 0; !ID_BLOCK_NOID(idl, i) && id > ID_BLOCK_ID(idl, i); i++ )
 554                 ;       /* NULL */
 555 #else
 556         i = idl_find(idl, id);
 557         if (ID_BLOCK_ID(idl, i) < id)
 558                 i++;
 559 #endif
 560
 561         if ( i != 0 ) {
 562                 i--;
 563                 first = 0;
 564         } else {
 565                 first = 1;
 566         }
 567
 568         /* get the block */
 569         cont_alloc( &k2, &key );
 570         cont_id( &k2, ID_BLOCK_ID(idl, i) );
 571
 572         if ( (tmp = idl_fetch_one( be, db, k2 )) == NULL ) {
 573 #ifdef NEW_LOGGING
 574                 LDAP_LOG( INDEX, ERR,
 575                            "idl_insert_key: nonexistent continuation block\n", 0, 0, 0 );
 576 #else
 577                 Debug( LDAP_DEBUG_ANY, "idl_insert_key: nonexistent continuation block\n",
 578                     0, 0, 0 );
 579 #endif
 580
 581                 cont_free( &k2 );
 582                 idl_free( idl );
 583                 return( -1 );
 584         }
 585
 586         /* insert the id */
 587         switch ( idl_insert( &tmp, id, db->dbc_maxids ) ) {
 588         case 0:         /* id inserted ok */
 589                 if ( (rc = idl_store( be, db, k2, tmp )) != 0 ) {
 590 #ifdef NEW_LOGGING
 591                         LDAP_LOG( INDEX, ERR,
 592                                    "ids_insert_key: idl_store returned %d\n", rc, 0, 0 );
 593 #else
 594                         Debug( LDAP_DEBUG_ANY,
 595                             "idl_insert_key: idl_store returned %d\n", rc, 0, 0 );
 596 #endif
 597
 598                 }
 599                 break;
 600
 601         case 1:         /* id inserted - first id in block has changed */
 602                 /*
 603                  * key for this block has changed, so we have to
 604                  * write the block under the new key, delete the
 605                  * old key block + update and write the indirect
 606                  * header block.
 607                  */
 608
 609                 rc = idl_change_first( be, db, key, idl, i, k2, tmp );
 610                 break;
 611
 612         case 2:         /* id not inserted - already there, do nothing */
 613                 rc = 0;
 614                 break;
 615
 616         case 3:         /* id not inserted - block is full */
 617                 /*
 618                  * first, see if it will fit in the next block,
 619                  * without splitting, unless we're trying to insert
 620                  * into the beginning of the first block.
 621                  */
 622
 623 #ifndef USE_INDIRECT_NIDS
 624                 /* is there a next block? */
 625                 if ( !first && !ID_BLOCK_NOID(idl, i + 1) ) {
 626 #else
 627                 if ( !first && (unsigned long)(i + 1) < ID_BLOCK_NIDS(idl) ) {
 628 #endif
 629                         /* read it in */
 630                         cont_alloc( &k2, &key );
 631                         cont_id( &k2, ID_BLOCK_ID(idl, i) );
 632                         if ( (tmp2 = idl_fetch_one( be, db, k2 )) == NULL ) {
 633 #ifdef NEW_LOGGING
 634                                 LDAP_LOG( INDEX, ERR,
 635                                            "idl_insert_key: idl_fetch_one returned NULL\n", 0, 0, 0);
 636 #else
 637                                 Debug( LDAP_DEBUG_ANY,
 638                                     "idl_insert_key: idl_fetch_one returned NULL\n",
 639                                     0, 0, 0 );
 640 #endif
 641
 642                                 /* split the original block */
 643                                 cont_free( &k2 );
 644                                 goto split;
 645                         }
 646
 647                         /* If the new id is less than the last id in the
 648                          * current block, it must not be put into the next
 649                          * block. Push the last id of the current block
 650                          * into the next block instead.
 651                          */
 652                         if (id < ID_BLOCK_ID(tmp, ID_BLOCK_NIDS(tmp) - 1)) {
 653                             ID id2 = ID_BLOCK_ID(tmp, ID_BLOCK_NIDS(tmp) - 1);
 654
 655                             --ID_BLOCK_NIDS(tmp);
 656                             /* This must succeed since we just popped one
 657                              * ID off the end of it.
 658                              */
 659                             rc = idl_insert( &tmp, id, db->dbc_maxids );
 660
 661                             if ( (rc = idl_store( be, db, k2, tmp )) != 0 ) {
 662 #ifdef NEW_LOGGING
 663                                 LDAP_LOG( INDEX, ERR,
 664                                         "idl_insert_key: idl_store returned %d\n", rc, 0, 0 );
 665 #else
 666                                 Debug( LDAP_DEBUG_ANY,
 667                             "idl_insert_key: idl_store returned %d\n", rc, 0, 0 );
 668 #endif
 669
 670                             }
 671
 672                             id = id2;
 673                             /* This new id will necessarily be inserted
 674                              * as the first id of the next block by the
 675                              * following switch() statement.
 676                              */
 677                         }
 678
 679                         switch ( (rc = idl_insert( &tmp2, id,
 680                             db->dbc_maxids )) ) {
 681                         case 1:         /* id inserted first in block */
 682                                 rc = idl_change_first( be, db, key, idl,
 683                                     i + 1, k2, tmp2 );
 684                                 /* FALL */
 685
 686                         case 2:         /* id already there - how? */
 687                         case 0:         /* id inserted: this can never be
 688                                          * the result of idl_insert, because
 689                                          * we guaranteed that idl_change_first
 690                                          * will always be called.
 691                                          */
 692                                 if ( rc == 2 ) {
 693 #ifdef NEW_LOGGING
 694                                         LDAP_LOG( INDEX, INFO,
 695                                                    "idl_insert_key: id %ld is already in next block\n",
 696                                                    id, 0, 0 );
 697 #else
 698                                         Debug( LDAP_DEBUG_ANY,
 699                                             "idl_insert_key: id %ld already in next block\n",
 700                                             id, 0, 0 );
 701 #endif
 702
 703                                 }
 704
 705                                 idl_free( tmp );
 706                                 idl_free( tmp2 );
 707                                 idl_free( idl );
 708                                 return( 0 );
 709
 710                         case 3:         /* split the original block */
 711                                 break;
 712                         }
 713
 714                         idl_free( tmp2 );
 715                 }
 716
 717 split:
 718                 /*
 719                  * must split the block, write both new blocks + update
 720                  * and write the indirect header block.
 721                  */
 722
 723                 rc = 0; /* optimistic */
 724
 725
 726 #ifndef USE_INDIRECT_NIDS
 727                 /* count how many indirect blocks *//* XXX linear count XXX */
 728                 for ( j = 0; !ID_BLOCK_NOID(idl, j); j++ )
 729                         ;       /* NULL */
 730 #else
 731                 j = ID_BLOCK_NIDS(idl);
 732 #endif
 733
 734                 /* check it against all-id thresholed */
 735                 if ( j + 1 > db->dbc_maxindirect ) {
 736                         /*
 737                          * we've passed the all-id threshold, meaning
 738                          * that this set of blocks should be replaced
 739                          * by a single "all-id" block.  our job: delete
 740                          * all the indirect blocks, and replace the header
 741                          * block by an all-id block.
 742                          */
 743
 744                         /* delete all indirect blocks */
 745 #ifndef USE_INDIRECT_NIDS
 746                         for ( j = 0; !ID_BLOCK_NOID(idl, j); j++ ) {
 747 #else
 748                         for ( j = 0; (unsigned long) j < ID_BLOCK_NIDS(idl); j++ ) {
 749 #endif
 750                                 cont_id( &k2, ID_BLOCK_ID(idl, j) );
 751
 752                                 rc = ldbm_cache_delete( db, k2 );
 753                         }
 754
 755                         /* store allid block in place of header block */
 756                         idl_free( idl );
 757                         idl = idl_allids( be );
 758                         rc = idl_store( be, db, key, idl );
 759
 760                         cont_free( &k2 );
 761                         idl_free( idl );
 762                         idl_free( tmp );
 763                         return( rc );
 764                 }
 765
 766                 idl_split_block( tmp, id, &tmp2, &tmp3 );
 767                 idl_free( tmp );
 768
 769                 /* create a new updated indirect header block */
 770                 tmp = idl_alloc( ID_BLOCK_NMAXN(idl) + 1 );
 771 #ifndef USE_INDIRECT_NIDS
 772                 ID_BLOCK_NIDS(tmp) = ID_BLOCK_INDIRECT_VALUE;
 773 #else
 774                 ID_BLOCK_NMAX(tmp) |= ID_BLOCK_INDIRECT_VALUE;
 775 #endif
 776                 /* everything up to the split block */
 777                 AC_MEMCPY(
 778                         (char *) &ID_BLOCK_ID(tmp, 0),
 779                         (char *) &ID_BLOCK_ID(idl, 0),
 780                     i * sizeof(ID) );
 781                 /* the two new blocks */
 782                 ID_BLOCK_ID(tmp, i) = ID_BLOCK_ID(tmp2, 0);
 783                 ID_BLOCK_ID(tmp, i + 1) = ID_BLOCK_ID(tmp3, 0);
 784                 /* everything after the split block */
 785 #ifndef USE_INDIRECT_NIDS
 786                 AC_MEMCPY(
 787                         (char *) &ID_BLOCK_ID(tmp, i + 2),
 788                         (char *) &ID_BLOCK_ID(idl, i + 1),
 789                         (ID_BLOCK_NMAXN(idl) - i - 1) * sizeof(ID) );
 790 #else
 791                 AC_MEMCPY(
 792                         (char *) &ID_BLOCK_ID(tmp, i + 2),
 793                         (char *) &ID_BLOCK_ID(idl, i + 1),
 794                         (ID_BLOCK_NIDS(idl) - i - 1) * sizeof(ID) );
 795                 ID_BLOCK_NIDS(tmp) = ID_BLOCK_NIDS(idl) + 1;
 796 #endif
 797
 798                 /* store the header block */
 799                 rc = idl_store( be, db, key, tmp );
 800
 801                 /* store the first id block */
 802                 cont_id( &k2, ID_BLOCK_ID(tmp2, 0) );
 803                 rc = idl_store( be, db, k2, tmp2 );
 804
 805                 /* store the second id block */
 806                 cont_id( &k2, ID_BLOCK_ID(tmp3, 0) );
 807                 rc = idl_store( be, db, k2, tmp3 );
 808
 809                 idl_free( tmp2 );
 810                 idl_free( tmp3 );
 811                 break;
 812         }
 813
 814         cont_free( &k2 );
 815         idl_free( tmp );
 816         idl_free( idl );
 817         return( rc );
 818 }
 819
 820
 821 /*
 822  * idl_insert - insert an id into an id list.
 823  *
 824  *      returns
 825  *              0       id inserted
 826  *              1       id inserted, first id in block has changed
 827  *              2       id not inserted, already there
 828  *              3       id not inserted, block must be split
 829  */
 830 int
 831 idl_insert( ID_BLOCK **idl, ID id, unsigned int maxids )
 832 {
 833         unsigned int    i;
 834
 835         if ( ID_BLOCK_ALLIDS( *idl ) ) {
 836                 return( 2 );    /* already there */
 837         }
 838
 839         /* is it already there? */
 840         i = idl_find(*idl, id);
 841         if ( ID_BLOCK_ID(*idl, i) == id ) {
 842                 return( 2 );    /* already there */
 843         }
 844         if ( ID_BLOCK_NIDS(*idl) && ID_BLOCK_ID(*idl, i) < id )
 845                 i++;
 846
 847         /* do we need to make room for it? */
 848         if ( ID_BLOCK_NIDS(*idl) == ID_BLOCK_NMAXN(*idl) ) {
 849                 /* make room or indicate block needs splitting */
 850                 if ( ID_BLOCK_NMAXN(*idl) >= maxids ) {
 851                         return( 3 );    /* block needs splitting */
 852                 }
 853
 854                 ID_BLOCK_NMAX(*idl) *= 2;
 855                 if ( ID_BLOCK_NMAXN(*idl) > maxids ) {
 856                         ID_BLOCK_NMAX(*idl) = maxids;
 857                 }
 858                 *idl = (ID_BLOCK *) ch_realloc( (char *) *idl,
 859                     (ID_BLOCK_NMAXN(*idl) + ID_BLOCK_IDS_OFFSET) * sizeof(ID) );
 860         }
 861
 862         /* make a slot for the new id */
 863         AC_MEMCPY( &ID_BLOCK_ID(*idl, i+1), &ID_BLOCK_ID(*idl, i),
 864                     (ID_BLOCK_NIDS(*idl) - i) * sizeof(ID) );
 865
 866         ID_BLOCK_ID(*idl, i) = id;
 867         ID_BLOCK_NIDS(*idl)++;
 868         (void) memset(
 869                 (char *) &ID_BLOCK_ID((*idl), ID_BLOCK_NIDS(*idl)),
 870                 '\0',
 871             (ID_BLOCK_NMAXN(*idl) - ID_BLOCK_NIDS(*idl)) * sizeof(ID) );
 872
 873 #ifdef LDBM_DEBUG_IDL
 874         idl_check(*idl);
 875 #endif
 876
 877         return( i == 0 ? 1 : 0 );       /* inserted - first id changed or not */
 878 }
 879
 880
 881 int
 882 idl_delete_key (
 883         Backend         *be,
 884         DBCache  *db,
 885         Datum           key,
 886         ID              id
 887 )
 888 {
 889         Datum  data;
 890         ID_BLOCK *idl;
 891         unsigned i;
 892         int j, nids;
 893
 894         if ( (idl = idl_fetch_one( be, db, key ) ) == NULL )
 895         {
 896                 /* It wasn't found.  Hmm... */
 897                 return -1;
 898         }
 899
 900         if ( ID_BLOCK_ALLIDS( idl ) ) {
 901                 idl_free( idl );
 902                 return 0;
 903         }
 904
 905         if ( ! ID_BLOCK_INDIRECT( idl ) ) {
 906                 i = idl_find(idl, id);
 907                 if ( ID_BLOCK_ID(idl, i) == id ) {
 908                         if( --ID_BLOCK_NIDS(idl) == 0 ) {
 909                                 ldbm_cache_delete( db, key );
 910
 911                         } else {
 912                                 AC_MEMCPY(
 913                                         &ID_BLOCK_ID(idl, i),
 914                                         &ID_BLOCK_ID(idl, i+1),
 915                                         (ID_BLOCK_NIDS(idl)-i) * sizeof(ID) );
 916
 917                                 ID_BLOCK_ID(idl, ID_BLOCK_NIDS(idl)) = NOID;
 918
 919                                 idl_store( be, db, key, idl );
 920                         }
 921
 922                         idl_free( idl );
 923                         return 0;
 924                 }
 925                 /*  We didn't find the ID.  Hmmm... */
 926                 idl_free( idl );
 927                 return -1;
 928         }
 929
 930         /* We have to go through an indirect block and find the ID
 931            in the list of IDL's
 932            */
 933         cont_alloc( &data, &key );
 934 #ifndef USE_INDIRECT_NIDS
 935         for ( nids = 0; !ID_BLOCK_NOID(idl, nids); nids++ )
 936                 ;       /* NULL */
 937
 938         for ( j = 0; j<nids; j++ )
 939 #else
 940         nids = ID_BLOCK_NIDS(idl);
 941         for ( j = idl_find(idl, id); j >= 0; j = -1)    /* execute once */
 942 #endif
 943         {
 944                 ID_BLOCK *tmp;
 945                 cont_id( &data, ID_BLOCK_ID(idl, j) );
 946
 947                 if ( (tmp = idl_fetch_one( be, db, data )) == NULL ) {
 948 #ifdef NEW_LOGGING
 949                         LDAP_LOG( INDEX, INFO,
 950                                    "idl_delete_key: idl_fetch_one returned NULL\n", 0, 0, 0 );
 951 #else
 952                         Debug( LDAP_DEBUG_ANY,
 953                             "idl_delete_key: idl_fetch of returned NULL\n", 0, 0, 0 );
 954 #endif
 955
 956                         continue;
 957                 }
 958                 /*
 959                    Now try to find the ID in tmp
 960                 */
 961
 962                 i = idl_find(tmp, id);
 963                 if ( ID_BLOCK_ID(tmp, i) == id )
 964                 {
 965                         AC_MEMCPY(
 966                                 &ID_BLOCK_ID(tmp, i),
 967                                 &ID_BLOCK_ID(tmp, i+1),
 968                                 (ID_BLOCK_NIDS(tmp)-(i+1)) * sizeof(ID));
 969                         ID_BLOCK_ID(tmp, ID_BLOCK_NIDS(tmp)-1 ) = NOID;
 970                         ID_BLOCK_NIDS(tmp)--;
 971
 972                         if ( ID_BLOCK_NIDS(tmp) ) {
 973                                 idl_store ( be, db, data, tmp );
 974
 975                         } else {
 976                                 ldbm_cache_delete( db, data );
 977                                 AC_MEMCPY(
 978                                         &ID_BLOCK_ID(idl, j),
 979                                         &ID_BLOCK_ID(idl, j+1),
 980                                         (nids-(j+1)) * sizeof(ID));
 981                                 ID_BLOCK_ID(idl, nids-1) = NOID;
 982                                 nids--;
 983 #ifdef USE_INDIRECT_NIDS
 984                                 ID_BLOCK_NIDS(idl)--;
 985 #endif
 986                                 if ( ! nids )
 987                                         ldbm_cache_delete( db, key );
 988                                 else
 989                                         idl_store( be, db, key, idl );
 990                         }
 991                         idl_free( tmp );
 992                         cont_free( &data );
 993                         idl_free( idl );
 994                         return 0;
 995                 }
 996                 idl_free( tmp );
 997         }
 998
 999         cont_free( &data );
1000         idl_free( idl );
1001         return -1;
1002 }
1003
1004
1005 /* return a duplicate of a single ID_BLOCK */
1006 static ID_BLOCK *
1007 idl_dup( ID_BLOCK *idl )
1008 {
1009         ID_BLOCK        *new;
1010
1011         if ( idl == NULL ) {
1012                 return( NULL );
1013         }
1014
1015         new = idl_alloc( ID_BLOCK_NMAXN(idl) );
1016
1017         AC_MEMCPY(
1018                 (char *) new,
1019                 (char *) idl,
1020                 (ID_BLOCK_NMAXN(idl) + ID_BLOCK_IDS_OFFSET) * sizeof(ID) );
1021
1022 #ifdef LDBM_DEBUG_IDL
1023         idl_check(new);
1024 #endif
1025
1026         return( new );
1027 }
1028
1029
1030 /* return the smaller ID_BLOCK */
1031 static ID_BLOCK *
1032 idl_min( ID_BLOCK *a, ID_BLOCK *b )
1033 {
1034         return( ID_BLOCK_NIDS(a) > ID_BLOCK_NIDS(b) ? b : a );
1035 }
1036
1037
1038 /*
1039  * idl_intersection - return a intersection b
1040  */
1041 ID_BLOCK *
1042 idl_intersection(
1043     Backend     *be,
1044     ID_BLOCK    *a,
1045     ID_BLOCK    *b
1046 )
1047 {
1048         unsigned int    ai, bi, ni;
1049         ID_BLOCK                *n;
1050
1051         if ( a == NULL || b == NULL ) {
1052                 return( NULL );
1053         }
1054         if ( ID_BLOCK_ALLIDS( a ) ) {
1055                 return( idl_dup( b ) );
1056         }
1057         if ( ID_BLOCK_ALLIDS( b ) ) {
1058                 return( idl_dup( a ) );
1059         }
1060         if ( ID_BLOCK_NIDS(a) == 0 || ID_BLOCK_NIDS(b) == 0 ) {
1061                 return( NULL );
1062         }
1063
1064         n = idl_dup( idl_min( a, b ) );
1065
1066 #ifdef LDBM_DEBUG_IDL
1067         idl_check(a);
1068         idl_check(b);
1069 #endif
1070
1071         for ( ni = 0, ai = 0, bi = 0; ; ) {
1072                 if ( ID_BLOCK_ID(b, bi) == ID_BLOCK_ID(a, ai) ) {
1073                         ID_BLOCK_ID(n, ni++) = ID_BLOCK_ID(a, ai);
1074                         ai++;
1075                         bi++;
1076                         if ( ai >= ID_BLOCK_NIDS(a) || bi >= ID_BLOCK_NIDS(b) )
1077                                 break;
1078                 } else if ( ID_BLOCK_ID(a, ai) < ID_BLOCK_ID(b, bi) ) {
1079                         ai++;
1080                         if ( ai >= ID_BLOCK_NIDS(a) )
1081                                 break;
1082                 } else {
1083                         bi++;
1084                         if ( bi >= ID_BLOCK_NIDS(b) )
1085                                 break;
1086                 }
1087         }
1088
1089         if ( ni == 0 ) {
1090                 idl_free( n );
1091                 return( NULL );
1092         }
1093         ID_BLOCK_NIDS(n) = ni;
1094
1095 #ifdef LDBM_DEBUG_IDL
1096         idl_check(n);
1097 #endif
1098
1099         return( n );
1100 }
1101
1102
1103 /*
1104  * idl_union - return a union b
1105  */
1106 ID_BLOCK *
1107 idl_union(
1108     Backend     *be,
1109     ID_BLOCK    *a,
1110     ID_BLOCK    *b
1111 )
1112 {
1113         unsigned int    ai, bi, ni;
1114         ID_BLOCK                *n;
1115
1116         if ( a == NULL ) {
1117                 return( idl_dup( b ) );
1118         }
1119         if ( b == NULL ) {
1120                 return( idl_dup( a ) );
1121         }
1122         if ( ID_BLOCK_ALLIDS( a ) || ID_BLOCK_ALLIDS( b ) ) {
1123                 return( idl_allids( be ) );
1124         }
1125
1126 #ifdef LDBM_DEBUG_IDL
1127         idl_check(a);
1128         idl_check(b);
1129 #endif
1130
1131         if ( ID_BLOCK_NIDS(b) < ID_BLOCK_NIDS(a) ) {
1132                 n = a;
1133                 a = b;
1134                 b = n;
1135         }
1136
1137         n = idl_alloc( ID_BLOCK_NIDS(a) + ID_BLOCK_NIDS(b) );
1138
1139         for ( ni = 0, ai = 0, bi = 0;
1140                 ai < ID_BLOCK_NIDS(a) && bi < ID_BLOCK_NIDS(b);
1141                 )
1142         {
1143                 if ( ID_BLOCK_ID(a, ai) < ID_BLOCK_ID(b, bi) ) {
1144                         ID_BLOCK_ID(n, ni++) = ID_BLOCK_ID(a, ai++);
1145
1146                 } else if ( ID_BLOCK_ID(b, bi) < ID_BLOCK_ID(a, ai) ) {
1147                         ID_BLOCK_ID(n, ni++) = ID_BLOCK_ID(b, bi++);
1148
1149                 } else {
1150                         ID_BLOCK_ID(n, ni++) = ID_BLOCK_ID(a, ai);
1151                         ai++, bi++;
1152                 }
1153         }
1154
1155         for ( ; ai < ID_BLOCK_NIDS(a); ai++ ) {
1156                 ID_BLOCK_ID(n, ni++) = ID_BLOCK_ID(a, ai);
1157         }
1158         for ( ; bi < ID_BLOCK_NIDS(b); bi++ ) {
1159                 ID_BLOCK_ID(n, ni++) = ID_BLOCK_ID(b, bi);
1160         }
1161         ID_BLOCK_NIDS(n) = ni;
1162
1163 #ifdef LDBM_DEBUG_IDL
1164         idl_check(n);
1165 #endif
1166
1167         return( n );
1168 }
1169
1170
1171 /*
1172  * idl_notin - return a intersection ~b (or a minus b)
1173  */
1174 ID_BLOCK *
1175 idl_notin(
1176     Backend     *be,
1177     ID_BLOCK    *a,
1178     ID_BLOCK    *b
1179 )
1180 {
1181         unsigned int    ni, ai, bi;
1182         ID_BLOCK                *n;
1183
1184         if ( a == NULL ) {
1185                 return( NULL );
1186         }
1187         if ( b == NULL || ID_BLOCK_ALLIDS( b )) {
1188                 return( idl_dup( a ) );
1189         }
1190
1191         if ( ID_BLOCK_ALLIDS( a ) ) {
1192                 n = idl_alloc( SLAPD_LDBM_MIN_MAXIDS );
1193                 ni = 0;
1194
1195                 for ( ai = 1, bi = 0;
1196                         ai < ID_BLOCK_NIDS(a) && ni < ID_BLOCK_NMAXN(n) && bi < ID_BLOCK_NMAXN(b);
1197                         ai++ )
1198                 {
1199                         if ( ID_BLOCK_ID(b, bi) == ai ) {
1200                                 bi++;
1201                         } else {
1202                                 ID_BLOCK_ID(n, ni++) = ai;
1203                         }
1204                 }
1205
1206                 for ( ; ai < ID_BLOCK_NIDS(a) && ni < ID_BLOCK_NMAXN(n); ai++ ) {
1207                         ID_BLOCK_ID(n, ni++) = ai;
1208                 }
1209
1210                 if ( ni == ID_BLOCK_NMAXN(n) ) {
1211                         idl_free( n );
1212                         return( idl_allids( be ) );
1213                 } else {
1214                         ID_BLOCK_NIDS(n) = ni;
1215                         return( n );
1216                 }
1217         }
1218
1219         n = idl_dup( a );
1220
1221         ni = 0;
1222         for ( ai = 0, bi = 0; ai < ID_BLOCK_NIDS(a); ai++ ) {
1223                 for ( ;
1224                         bi < ID_BLOCK_NIDS(b) && ID_BLOCK_ID(b, bi) < ID_BLOCK_ID(a, ai);
1225                     bi++ )
1226                 {
1227                         ;       /* NULL */
1228                 }
1229
1230                 if ( bi == ID_BLOCK_NIDS(b) ) {
1231                         break;
1232                 }
1233
1234                 if ( ID_BLOCK_ID(b, bi) != ID_BLOCK_ID(a, ai) ) {
1235                         ID_BLOCK_ID(n, ni++) = ID_BLOCK_ID(a, ai);
1236                 }
1237         }
1238
1239         for ( ; ai < ID_BLOCK_NIDS(a); ai++ ) {
1240                 ID_BLOCK_ID(n, ni++) = ID_BLOCK_ID(a, ai);
1241         }
1242         ID_BLOCK_NIDS(n) = ni;
1243
1244 #ifdef LDBM_DEBUG_IDL
1245         idl_check(n);
1246 #endif
1247
1248         return( n );
1249 }
1250
1251 /*      return the first ID in the block
1252  *      if ALLIDS block
1253  *              NIDS > 1 return 1
1254  *              otherwise return NOID
1255  *      otherwise return first ID
1256  *
1257  *      cursor is set to 1
1258  */
1259 ID
1260 idl_firstid( ID_BLOCK *idl, ID *cursor )
1261 {
1262         *cursor = 1;
1263
1264         if ( idl == NULL || ID_BLOCK_NIDS(idl) == 0 ) {
1265                 return( NOID );
1266         }
1267
1268         if ( ID_BLOCK_ALLIDS( idl ) ) {
1269                 return( ID_BLOCK_NIDS(idl) > 1 ? 1 : NOID );
1270         }
1271
1272         return( ID_BLOCK_ID(idl, 0) );
1273 }
1274
1275 /*      return next ID
1276  *      if ALLIDS block, cursor is id.
1277  *              increment id
1278  *              if id < NIDS return id
1279  *              otherwise NOID.
1280  *      otherwise cursor is index into block
1281  *              if index < nids
1282  *                      return id at index then increment
1283  */
1284 ID
1285 idl_nextid( ID_BLOCK *idl, ID *cursor )
1286 {
1287         if ( ID_BLOCK_ALLIDS( idl ) ) {
1288                 if( ++(*cursor) < ID_BLOCK_NIDS(idl) ) {
1289                         return *cursor;
1290                 } else {
1291                         return NOID;
1292                 }
1293         }
1294
1295         if ( *cursor < ID_BLOCK_NIDS(idl) ) {
1296                 return( ID_BLOCK_ID(idl, (*cursor)++) );
1297         }
1298
1299         return( NOID );
1300 }