git.sur5r.net Git - openldap/blob - servers/slapd/back-ldbm/idl.c

   1 /* idl.c - ldap id list handling routines */
   2 /* $OpenLDAP$ */
   3 /*
   4  * Copyright 1998-2003 The OpenLDAP Foundation, All Rights Reserved.
   5  * COPYING RESTRICTIONS APPLY, see COPYRIGHT file
   6  */
   7
   8 #include "portable.h"
   9
  10 #include <stdio.h>
  11
  12 #include <ac/string.h>
  13 #include <ac/socket.h>
  14
  15 #include "slap.h"
  16 #include "back-ldbm.h"
  17
  18 static ID_BLOCK* idl_dup( ID_BLOCK *idl );
  19
  20 static void cont_alloc( Datum *cont, Datum *key )
  21 {
  22         ldbm_datum_init( *cont );
  23         cont->dsize = 1 + sizeof(ID) + key->dsize;
  24         cont->dptr = ch_malloc( cont->dsize );
  25
  26         * (unsigned char *) cont->dptr = SLAP_INDEX_CONT_PREFIX;
  27
  28         AC_MEMCPY( &((unsigned char *)cont->dptr)[1 + sizeof(ID)],
  29                 key->dptr, key->dsize );
  30 }
  31
  32 static void cont_id( Datum *cont, ID id )
  33 {
  34         unsigned int i;
  35
  36         for( i=1; i <= sizeof(id); i++) {
  37                 ((unsigned char *)cont->dptr)[i] = (unsigned char)(id & 0xFF);
  38                 id >>= 8;
  39         }
  40
  41 }
  42
  43 static void cont_free( Datum *cont )
  44 {
  45         ch_free( cont->dptr );
  46 }
  47
  48 #ifdef LDBM_DEBUG_IDL
  49 static void idl_check(ID_BLOCK *idl)
  50 {
  51         int i, max;
  52         ID_BLOCK last;
  53
  54         if( ID_BLOCK_ALLIDS(idl) )
  55         {
  56                 return;
  57         }
  58 #ifndef USE_INDIRECT_NIDS
  59         if( ID_BLOCK_INDIRECT(idl) )
  60         {
  61                 for ( max = 0; !ID_BLOCK_NOID(idl, max); max++ ) ;
  62         } else
  63 #endif
  64         {
  65                 max = ID_BLOCK_NIDS(idl);
  66         }
  67         if ( max <= 1 )
  68         {
  69                 return;
  70         }
  71
  72         for( last = ID_BLOCK_ID(idl, 0), i = 1;
  73                 i < max;
  74                 last = ID_BLOCK_ID(idl, i), i++ )
  75         {
  76                 assert (last < ID_BLOCK_ID(idl, i) );
  77         }
  78 }
  79 #endif
  80
  81 /* Allocate an ID_BLOCK with room for nids ids */
  82 ID_BLOCK *
  83 idl_alloc( unsigned int nids )
  84 {
  85         ID_BLOCK        *new;
  86
  87         /* nmax + nids + space for the ids */
  88         new = (ID_BLOCK *) ch_calloc( (ID_BLOCK_IDS_OFFSET + nids), sizeof(ID) );
  89         ID_BLOCK_NMAX(new) = nids;
  90         ID_BLOCK_NIDS(new) = 0;
  91
  92         return( new );
  93 }
  94
  95
  96 /* Allocate an empty ALLIDS ID_BLOCK */
  97 ID_BLOCK        *
  98 idl_allids( Backend *be )
  99 {
 100         ID_BLOCK        *idl;
 101         ID              id;
 102
 103         idl = idl_alloc( 0 );
 104         ID_BLOCK_NMAX(idl) = ID_BLOCK_ALLIDS_VALUE;
 105         if ( next_id_get( be, &id ) ) {
 106                 return NULL;
 107         }
 108         ID_BLOCK_NIDS(idl) = id;
 109
 110         return( idl );
 111 }
 112
 113 /* Free an ID_BLOCK */
 114 void
 115 idl_free( ID_BLOCK *idl )
 116 {
 117         if ( idl == NULL ) {
 118 #ifdef NEW_LOGGING
 119                 LDAP_LOG( INDEX, INFO, "idl_free: called with NULL pointer\n" , 0,0,0);
 120 #else
 121                 Debug( LDAP_DEBUG_TRACE,
 122                         "idl_free: called with NULL pointer\n",
 123                         0, 0, 0 );
 124 #endif
 125
 126                 return;
 127         }
 128
 129         free( (char *) idl );
 130 }
 131
 132
 133 /* Fetch an single ID_BLOCK from the cache */
 134 static ID_BLOCK *
 135 idl_fetch_one(
 136     Backend             *be,
 137     DBCache     *db,
 138     Datum               key
 139 )
 140 {
 141         Datum   data;
 142         ID_BLOCK        *idl;
 143
 144         /* Debug( LDAP_DEBUG_TRACE, "=> idl_fetch_one\n", 0, 0, 0 ); */
 145
 146         data = ldbm_cache_fetch( db, key );
 147
 148         if( data.dptr == NULL ) {
 149                 return NULL;
 150         }
 151
 152         idl = (ID_BLOCK *) data.dptr;
 153         if ( ID_BLOCK_ALLIDS(idl) ) {
 154                 /* make sure we have the current value of highest id */
 155                 idl = idl_allids( be );
 156         } else {
 157                 idl = idl_dup((ID_BLOCK *) data.dptr);
 158         }
 159
 160         ldbm_datum_free( db->dbc_db, data );
 161
 162         return idl;
 163 }
 164
 165
 166 /* Fetch a set of ID_BLOCKs from the cache
 167  *      if not INDIRECT
 168  *              if block return is an ALLIDS block,
 169  *                      return an new ALLIDS block
 170  *              otherwise
 171  *                      return block
 172  *      construct super block from all blocks referenced by INDIRECT block
 173  *      return super block
 174  */
 175 ID_BLOCK *
 176 idl_fetch(
 177     Backend             *be,
 178     DBCache     *db,
 179     Datum               key
 180 )
 181 {
 182         Datum   data;
 183         ID_BLOCK        *idl;
 184         ID_BLOCK        **tmp;
 185         unsigned        i, nids, nblocks;
 186
 187         idl = idl_fetch_one( be, db, key );
 188
 189         if ( idl == NULL ) {
 190                 return NULL;
 191         }
 192
 193         if ( ID_BLOCK_ALLIDS(idl) ) {
 194                 /* all ids block */
 195                 return( idl );
 196         }
 197
 198         if ( ! ID_BLOCK_INDIRECT( idl ) ) {
 199                 /* regular block */
 200                 return( idl );
 201         }
 202
 203         /*
 204          * this is an indirect block which points to other blocks.
 205          * we need to read in all the blocks it points to and construct
 206          * a big id list containing all the ids, which we will return.
 207          */
 208
 209 #ifndef USE_INDIRECT_NIDS
 210         /* count the number of blocks & allocate space for pointers to them */
 211         for ( nblocks = 0; !ID_BLOCK_NOID(idl, nblocks); nblocks++ )
 212                 ;       /* NULL */
 213 #else
 214         nblocks = ID_BLOCK_NIDS(idl);
 215 #endif
 216         tmp = (ID_BLOCK **) ch_malloc( nblocks * sizeof(ID_BLOCK *) );
 217
 218         /* read in all the blocks */
 219         cont_alloc( &data, &key );
 220         nids = 0;
 221         for ( i = 0; i < nblocks; i++ ) {
 222                 cont_id( &data, ID_BLOCK_ID(idl, i) );
 223
 224                 if ( (tmp[i] = idl_fetch_one( be, db, data )) == NULL ) {
 225 #ifdef NEW_LOGGING
 226                         LDAP_LOG( INDEX, INFO,
 227                                    "idl_fetch: idl_fetch_one returned NULL\n", 0,0,0 );
 228 #else
 229                         Debug( LDAP_DEBUG_ANY,
 230                             "idl_fetch: one returned NULL\n", 0, 0, 0 );
 231 #endif
 232
 233                         continue;
 234                 }
 235
 236                 nids += ID_BLOCK_NIDS(tmp[i]);
 237         }
 238         cont_free( &data );
 239         idl_free( idl );
 240
 241         /* allocate space for the big block */
 242         idl = idl_alloc( nids );
 243         ID_BLOCK_NIDS(idl) = nids;
 244         nids = 0;
 245
 246         /* copy in all the ids from the component blocks */
 247         for ( i = 0; i < nblocks; i++ ) {
 248                 if ( tmp[i] == NULL ) {
 249                         continue;
 250                 }
 251
 252                 AC_MEMCPY(
 253                         (char *) &ID_BLOCK_ID(idl, nids),
 254                         (char *) &ID_BLOCK_ID(tmp[i], 0),
 255                         ID_BLOCK_NIDS(tmp[i]) * sizeof(ID) );
 256                 nids += ID_BLOCK_NIDS(tmp[i]);
 257
 258                 idl_free( tmp[i] );
 259         }
 260         free( (char *) tmp );
 261
 262 #ifdef LDBM_DEBUG_IDL
 263         idl_check(idl);
 264 #endif
 265
 266 #ifdef NEW_LOGGING
 267         LDAP_LOG( INDEX, ENTRY,
 268                    "idl_fetch: %ld ids (%ld max)\n",
 269                    ID_BLOCK_NIDS(idl), ID_BLOCK_NMAXN(idl), 0 );
 270 #else
 271         Debug( LDAP_DEBUG_TRACE, "<= idl_fetch %ld ids (%ld max)\n",
 272                ID_BLOCK_NIDS(idl), ID_BLOCK_NMAXN(idl), 0 );
 273 #endif
 274
 275         return( idl );
 276 }
 277
 278
 279 /* store a single block */
 280 static int
 281 idl_store(
 282     Backend             *be,
 283     DBCache     *db,
 284     Datum               key,
 285     ID_BLOCK            *idl
 286 )
 287 {
 288         int     rc, flags;
 289         Datum   data;
 290
 291 #ifdef LDBM_DEBUG_IDL
 292         idl_check(idl);
 293 #endif
 294
 295         ldbm_datum_init( data );
 296
 297         /* Debug( LDAP_DEBUG_TRACE, "=> idl_store\n", 0, 0, 0 ); */
 298
 299         data.dptr = (char *) idl;
 300         data.dsize = (ID_BLOCK_IDS_OFFSET + ID_BLOCK_NMAXN(idl)) * sizeof(ID);
 301
 302         flags = LDBM_REPLACE;
 303         rc = ldbm_cache_store( db, key, data, flags );
 304
 305         /* Debug( LDAP_DEBUG_TRACE, "<= idl_store %d\n", rc, 0, 0 ); */
 306         return( rc );
 307 }
 308
 309 /* Binary search for id in block, return index
 310  *    an index is always returned, even with no match. If no
 311  * match, the returned index is the insertion point.
 312  */
 313 static unsigned int
 314 idl_find(
 315     ID_BLOCK    *b,
 316     ID          id
 317 )
 318 {
 319         int lo=0, hi=ID_BLOCK_NIDS(b)-1, nr=0;
 320
 321         for (;lo<=hi;)
 322         {
 323             nr = ( lo + hi ) / 2;
 324             if (ID_BLOCK_ID(b, nr) == id)
 325                 break;
 326             if (ID_BLOCK_ID(b, nr) > id)
 327                 hi = nr - 1;
 328             else
 329                 lo = nr + 1;
 330         }
 331         return nr;
 332 }
 333
 334 /* split the block at id
 335  *      locate ID greater than or equal to id.
 336  */
 337 static void
 338 idl_split_block(
 339     ID_BLOCK    *b,
 340     ID          id,
 341     ID_BLOCK    **right,
 342     ID_BLOCK    **left
 343 )
 344 {
 345         unsigned int    nr, nl;
 346
 347         /* find where to split the block */
 348         nr = idl_find(b, id);
 349         if ( ID_BLOCK_ID(b,nr) < id )
 350                 nr++;
 351
 352         nl = ID_BLOCK_NIDS(b) - nr;
 353
 354         *right = idl_alloc( nr == 0 ? 1 : nr );
 355         *left = idl_alloc( nl + (nr == 0 ? 0 : 1));
 356
 357         /*
 358          * everything before the id being inserted in the first block
 359          * unless there is nothing, in which case the id being inserted
 360          * goes there.
 361          */
 362         if ( nr == 0 ) {
 363                 ID_BLOCK_NIDS(*right) = 1;
 364                 ID_BLOCK_ID(*right, 0) = id;
 365         } else {
 366                 AC_MEMCPY(
 367                         (char *) &ID_BLOCK_ID(*right, 0),
 368                         (char *) &ID_BLOCK_ID(b, 0),
 369                         nr * sizeof(ID) );
 370                 ID_BLOCK_NIDS(*right) = nr;
 371                 ID_BLOCK_ID(*left, 0) = id;
 372         }
 373
 374         /* the id being inserted & everything after in the second block */
 375         AC_MEMCPY(
 376                 (char *) &ID_BLOCK_ID(*left, (nr == 0 ? 0 : 1)),
 377             (char *) &ID_BLOCK_ID(b, nr),
 378                 nl * sizeof(ID) );
 379         ID_BLOCK_NIDS(*left) = nl + (nr == 0 ? 0 : 1);
 380
 381 #ifdef LDBM_DEBUG_IDL
 382         idl_check(*right);
 383         idl_check(*left);
 384 #endif
 385 }
 386
 387
 388 /*
 389  * idl_change_first - called when an indirect block's first key has
 390  * changed, meaning it needs to be stored under a new key, and the
 391  * header block pointing to it needs updating.
 392  */
 393 static int
 394 idl_change_first(
 395     Backend             *be,
 396     DBCache     *db,
 397     Datum               hkey,           /* header block key     */
 398     ID_BLOCK            *h,             /* header block         */
 399     int                 pos,            /* pos in h to update   */
 400     Datum               bkey,           /* data block key       */
 401     ID_BLOCK            *b              /* data block           */
 402 )
 403 {
 404         int     rc;
 405
 406         /* Debug( LDAP_DEBUG_TRACE, "=> idl_change_first\n", 0, 0, 0 ); */
 407
 408         /* delete old key block */
 409         if ( (rc = ldbm_cache_delete( db, bkey )) != 0 ) {
 410 #ifdef NEW_LOGGING
 411                 LDAP_LOG( INDEX, INFO,
 412                            "idl_change_first: ldbm_cache_delete returned %d\n", rc, 0, 0 );
 413 #else
 414                 Debug( LDAP_DEBUG_ANY,
 415                     "idl_change_first: ldbm_cache_delete returned %d\n",
 416                         rc, 0, 0 );
 417 #endif
 418
 419                 return( rc );
 420         }
 421
 422         /* write block with new key */
 423         cont_id( &bkey, ID_BLOCK_ID(b, 0) );
 424
 425         if ( (rc = idl_store( be, db, bkey, b )) != 0 ) {
 426 #ifdef NEW_LOGGING
 427                 LDAP_LOG( INDEX, INFO,
 428                            "idl_change_first: idl_store returned %d\n", rc, 0, 0 );
 429 #else
 430                 Debug( LDAP_DEBUG_ANY,
 431                     "idl_change_first: idl_store returned %d\n", rc, 0, 0 );
 432 #endif
 433
 434                 return( rc );
 435         }
 436
 437         /* update + write indirect header block */
 438         ID_BLOCK_ID(h, pos) = ID_BLOCK_ID(b, 0);
 439         if ( (rc = idl_store( be, db, hkey, h )) != 0 ) {
 440 #ifdef NEW_LOGGING
 441                 LDAP_LOG( INDEX, INFO,
 442                            "idl_change_first: idl_store returned %s\n", rc, 0, 0 );
 443 #else
 444                 Debug( LDAP_DEBUG_ANY,
 445                     "idl_change_first: idl_store returned %d\n", rc, 0, 0 );
 446 #endif
 447
 448                 return( rc );
 449         }
 450
 451         return( 0 );
 452 }
 453
 454
 455 int
 456 idl_insert_key(
 457     Backend             *be,
 458     DBCache     *db,
 459     Datum               key,
 460     ID                  id
 461 )
 462 {
 463         int     i, j, first, rc = 0;
 464         ID_BLOCK        *idl, *tmp, *tmp2, *tmp3;
 465         Datum   k2;
 466
 467         if ( (idl = idl_fetch_one( be, db, key )) == NULL ) {
 468                 idl = idl_alloc( 1 );
 469                 ID_BLOCK_ID(idl, ID_BLOCK_NIDS(idl)++) = id;
 470                 rc = idl_store( be, db, key, idl );
 471
 472                 idl_free( idl );
 473                 return( rc );
 474         }
 475
 476         if ( ID_BLOCK_ALLIDS( idl ) ) {
 477                 /* ALLIDS */
 478                 idl_free( idl );
 479                 return 0;
 480         }
 481
 482         if ( ! ID_BLOCK_INDIRECT( idl ) ) {
 483                 /* regular block */
 484                 switch ( idl_insert( &idl, id, db->dbc_maxids ) ) {
 485                 case 0:         /* id inserted - store the updated block */
 486                 case 1:
 487                         rc = idl_store( be, db, key, idl );
 488                         break;
 489
 490                 case 2:         /* id already there - nothing to do */
 491                         rc = 0;
 492                         break;
 493
 494                 case 3:         /* id not inserted - block must be split */
 495                         /* check threshold for marking this an all-id block */
 496                         if ( db->dbc_maxindirect < 2 ) {
 497                                 idl_free( idl );
 498                                 idl = idl_allids( be );
 499                                 rc = idl_store( be, db, key, idl );
 500                                 break;
 501                         }
 502
 503                         idl_split_block( idl, id, &tmp, &tmp2 );
 504                         idl_free( idl );
 505
 506                         /* create the header indirect block */
 507 #ifndef USE_INDIRECT_NIDS
 508                         idl = idl_alloc( 3 );
 509                         ID_BLOCK_NMAX(idl) = 3;
 510                         ID_BLOCK_NIDS(idl) = ID_BLOCK_INDIRECT_VALUE;
 511                         ID_BLOCK_ID(idl, 0) = ID_BLOCK_ID(tmp, 0);
 512                         ID_BLOCK_ID(idl, 1) = ID_BLOCK_ID(tmp2, 0);
 513                         ID_BLOCK_ID(idl, 2) = NOID;
 514 #else
 515                         idl = idl_alloc( 2 );
 516                         ID_BLOCK_NMAX(idl) = 2 | ID_BLOCK_INDIRECT_VALUE;
 517                         ID_BLOCK_NIDS(idl) = 2;
 518                         ID_BLOCK_ID(idl, 0) = ID_BLOCK_ID(tmp, 0);
 519                         ID_BLOCK_ID(idl, 1) = ID_BLOCK_ID(tmp2, 0);
 520 #endif
 521
 522                         /* store it */
 523                         rc = idl_store( be, db, key, idl );
 524
 525                         cont_alloc( &k2, &key );
 526                         cont_id( &k2, ID_BLOCK_ID(tmp, 0) );
 527
 528                         rc = idl_store( be, db, k2, tmp );
 529
 530                         cont_id( &k2, ID_BLOCK_ID(tmp2, 0) );
 531                         rc = idl_store( be, db, k2, tmp2 );
 532
 533                         cont_free( &k2 );
 534
 535                         idl_free( tmp );
 536                         idl_free( tmp2 );
 537                         break;
 538                 }
 539
 540                 idl_free( idl );
 541                 return( rc );
 542         }
 543
 544         /*
 545          * this is an indirect block which points to other blocks.
 546          * we need to read in the block into which the id should be
 547          * inserted, then insert the id and store the block.  we might
 548          * have to split the block if it is full, which means we also
 549          * need to write a new "header" block.
 550          */
 551
 552 #ifndef USE_INDIRECT_NIDS
 553         /* select the block to try inserting into *//* XXX linear search XXX */
 554         for ( i = 0; !ID_BLOCK_NOID(idl, i) && id > ID_BLOCK_ID(idl, i); i++ )
 555                 ;       /* NULL */
 556 #else
 557         i = idl_find(idl, id);
 558         if (ID_BLOCK_ID(idl, i) < id)
 559                 i++;
 560 #endif
 561
 562         if ( i != 0 ) {
 563                 i--;
 564                 first = 0;
 565         } else {
 566                 first = 1;
 567         }
 568
 569         /* get the block */
 570         cont_alloc( &k2, &key );
 571         cont_id( &k2, ID_BLOCK_ID(idl, i) );
 572
 573         if ( (tmp = idl_fetch_one( be, db, k2 )) == NULL ) {
 574 #ifdef NEW_LOGGING
 575                 LDAP_LOG( INDEX, ERR,
 576                            "idl_insert_key: nonexistent continuation block\n", 0, 0, 0 );
 577 #else
 578                 Debug( LDAP_DEBUG_ANY, "idl_insert_key: nonexistent continuation block\n",
 579                     0, 0, 0 );
 580 #endif
 581
 582                 cont_free( &k2 );
 583                 idl_free( idl );
 584                 return( -1 );
 585         }
 586
 587         /* insert the id */
 588         switch ( idl_insert( &tmp, id, db->dbc_maxids ) ) {
 589         case 0:         /* id inserted ok */
 590                 if ( (rc = idl_store( be, db, k2, tmp )) != 0 ) {
 591 #ifdef NEW_LOGGING
 592                         LDAP_LOG( INDEX, ERR,
 593                                    "ids_insert_key: idl_store returned %d\n", rc, 0, 0 );
 594 #else
 595                         Debug( LDAP_DEBUG_ANY,
 596                             "idl_insert_key: idl_store returned %d\n", rc, 0, 0 );
 597 #endif
 598
 599                 }
 600                 break;
 601
 602         case 1:         /* id inserted - first id in block has changed */
 603                 /*
 604                  * key for this block has changed, so we have to
 605                  * write the block under the new key, delete the
 606                  * old key block + update and write the indirect
 607                  * header block.
 608                  */
 609
 610                 rc = idl_change_first( be, db, key, idl, i, k2, tmp );
 611                 break;
 612
 613         case 2:         /* id not inserted - already there, do nothing */
 614                 rc = 0;
 615                 break;
 616
 617         case 3:         /* id not inserted - block is full */
 618                 /*
 619                  * first, see if it will fit in the next block,
 620                  * without splitting, unless we're trying to insert
 621                  * into the beginning of the first block.
 622                  */
 623
 624 #ifndef USE_INDIRECT_NIDS
 625                 /* is there a next block? */
 626                 if ( !first && !ID_BLOCK_NOID(idl, i + 1) ) {
 627 #else
 628                 if ( !first && (unsigned long)(i + 1) < ID_BLOCK_NIDS(idl) ) {
 629 #endif
 630                         /* read it in */
 631                         cont_alloc( &k2, &key );
 632                         cont_id( &k2, ID_BLOCK_ID(idl, i + 1) );
 633                         if ( (tmp2 = idl_fetch_one( be, db, k2 )) == NULL ) {
 634 #ifdef NEW_LOGGING
 635                                 LDAP_LOG( INDEX, ERR,
 636                                            "idl_insert_key: idl_fetch_one returned NULL\n", 0, 0, 0);
 637 #else
 638                                 Debug( LDAP_DEBUG_ANY,
 639                                     "idl_insert_key: idl_fetch_one returned NULL\n",
 640                                     0, 0, 0 );
 641 #endif
 642
 643                                 /* split the original block */
 644                                 cont_free( &k2 );
 645                                 goto split;
 646                         }
 647
 648                         /* If the new id is less than the last id in the
 649                          * current block, it must not be put into the next
 650                          * block. Push the last id of the current block
 651                          * into the next block instead.
 652                          */
 653                         if (id < ID_BLOCK_ID(tmp, ID_BLOCK_NIDS(tmp) - 1)) {
 654                             ID id2 = ID_BLOCK_ID(tmp, ID_BLOCK_NIDS(tmp) - 1);
 655
 656                             --ID_BLOCK_NIDS(tmp);
 657                             /* This must succeed since we just popped one
 658                              * ID off the end of it.
 659                              */
 660                             rc = idl_insert( &tmp, id, db->dbc_maxids );
 661
 662                             if ( (rc = idl_store( be, db, k2, tmp )) != 0 ) {
 663 #ifdef NEW_LOGGING
 664                                 LDAP_LOG( INDEX, ERR,
 665                                         "idl_insert_key: idl_store returned %d\n", rc, 0, 0 );
 666 #else
 667                                 Debug( LDAP_DEBUG_ANY,
 668                             "idl_insert_key: idl_store returned %d\n", rc, 0, 0 );
 669 #endif
 670
 671                             }
 672
 673                             id = id2;
 674                             /* This new id will necessarily be inserted
 675                              * as the first id of the next block by the
 676                              * following switch() statement.
 677                              */
 678                         }
 679
 680                         switch ( (rc = idl_insert( &tmp2, id,
 681                             db->dbc_maxids )) ) {
 682                         case 1:         /* id inserted first in block */
 683                                 rc = idl_change_first( be, db, key, idl,
 684                                     i + 1, k2, tmp2 );
 685                                 /* FALL */
 686
 687                         case 2:         /* id already there - how? */
 688                         case 0:         /* id inserted: this can never be
 689                                          * the result of idl_insert, because
 690                                          * we guaranteed that idl_change_first
 691                                          * will always be called.
 692                                          */
 693                                 if ( rc == 2 ) {
 694 #ifdef NEW_LOGGING
 695                                         LDAP_LOG( INDEX, INFO,
 696                                                    "idl_insert_key: id %ld is already in next block\n",
 697                                                    id, 0, 0 );
 698 #else
 699                                         Debug( LDAP_DEBUG_ANY,
 700                                             "idl_insert_key: id %ld already in next block\n",
 701                                             id, 0, 0 );
 702 #endif
 703
 704                                 }
 705
 706                                 idl_free( tmp );
 707                                 idl_free( tmp2 );
 708                                 idl_free( idl );
 709                                 return( 0 );
 710
 711                         case 3:         /* split the original block */
 712                                 break;
 713                         }
 714
 715                         idl_free( tmp2 );
 716                 }
 717
 718 split:
 719                 /*
 720                  * must split the block, write both new blocks + update
 721                  * and write the indirect header block.
 722                  */
 723
 724                 rc = 0; /* optimistic */
 725
 726
 727 #ifndef USE_INDIRECT_NIDS
 728                 /* count how many indirect blocks *//* XXX linear count XXX */
 729                 for ( j = 0; !ID_BLOCK_NOID(idl, j); j++ )
 730                         ;       /* NULL */
 731 #else
 732                 j = ID_BLOCK_NIDS(idl);
 733 #endif
 734
 735                 /* check it against all-id thresholed */
 736                 if ( j + 1 > db->dbc_maxindirect ) {
 737                         /*
 738                          * we've passed the all-id threshold, meaning
 739                          * that this set of blocks should be replaced
 740                          * by a single "all-id" block.  our job: delete
 741                          * all the indirect blocks, and replace the header
 742                          * block by an all-id block.
 743                          */
 744
 745                         /* delete all indirect blocks */
 746 #ifndef USE_INDIRECT_NIDS
 747                         for ( j = 0; !ID_BLOCK_NOID(idl, j); j++ ) {
 748 #else
 749                         for ( j = 0; (unsigned long) j < ID_BLOCK_NIDS(idl); j++ ) {
 750 #endif
 751                                 cont_id( &k2, ID_BLOCK_ID(idl, j) );
 752
 753                                 rc = ldbm_cache_delete( db, k2 );
 754                         }
 755
 756                         /* store allid block in place of header block */
 757                         idl_free( idl );
 758                         idl = idl_allids( be );
 759                         rc = idl_store( be, db, key, idl );
 760
 761                         cont_free( &k2 );
 762                         idl_free( idl );
 763                         idl_free( tmp );
 764                         return( rc );
 765                 }
 766
 767                 idl_split_block( tmp, id, &tmp2, &tmp3 );
 768                 idl_free( tmp );
 769
 770                 /* create a new updated indirect header block */
 771                 tmp = idl_alloc( ID_BLOCK_NMAXN(idl) + 1 );
 772 #ifndef USE_INDIRECT_NIDS
 773                 ID_BLOCK_NIDS(tmp) = ID_BLOCK_INDIRECT_VALUE;
 774 #else
 775                 ID_BLOCK_NMAX(tmp) |= ID_BLOCK_INDIRECT_VALUE;
 776 #endif
 777                 /* everything up to the split block */
 778                 AC_MEMCPY(
 779                         (char *) &ID_BLOCK_ID(tmp, 0),
 780                         (char *) &ID_BLOCK_ID(idl, 0),
 781                     i * sizeof(ID) );
 782                 /* the two new blocks */
 783                 ID_BLOCK_ID(tmp, i) = ID_BLOCK_ID(tmp2, 0);
 784                 ID_BLOCK_ID(tmp, i + 1) = ID_BLOCK_ID(tmp3, 0);
 785                 /* everything after the split block */
 786 #ifndef USE_INDIRECT_NIDS
 787                 AC_MEMCPY(
 788                         (char *) &ID_BLOCK_ID(tmp, i + 2),
 789                         (char *) &ID_BLOCK_ID(idl, i + 1),
 790                         (ID_BLOCK_NMAXN(idl) - i - 1) * sizeof(ID) );
 791 #else
 792                 AC_MEMCPY(
 793                         (char *) &ID_BLOCK_ID(tmp, i + 2),
 794                         (char *) &ID_BLOCK_ID(idl, i + 1),
 795                         (ID_BLOCK_NIDS(idl) - i - 1) * sizeof(ID) );
 796                 ID_BLOCK_NIDS(tmp) = ID_BLOCK_NIDS(idl) + 1;
 797 #endif
 798
 799                 /* store the header block */
 800                 rc = idl_store( be, db, key, tmp );
 801
 802                 /* store the first id block */
 803                 cont_id( &k2, ID_BLOCK_ID(tmp2, 0) );
 804                 rc = idl_store( be, db, k2, tmp2 );
 805
 806                 /* store the second id block */
 807                 cont_id( &k2, ID_BLOCK_ID(tmp3, 0) );
 808                 rc = idl_store( be, db, k2, tmp3 );
 809
 810                 idl_free( tmp2 );
 811                 idl_free( tmp3 );
 812                 break;
 813         }
 814
 815         cont_free( &k2 );
 816         idl_free( tmp );
 817         idl_free( idl );
 818         return( rc );
 819 }
 820
 821
 822 /*
 823  * idl_insert - insert an id into an id list.
 824  *
 825  *      returns
 826  *              0       id inserted
 827  *              1       id inserted, first id in block has changed
 828  *              2       id not inserted, already there
 829  *              3       id not inserted, block must be split
 830  */
 831 int
 832 idl_insert( ID_BLOCK **idl, ID id, unsigned int maxids )
 833 {
 834         unsigned int    i;
 835
 836         if ( ID_BLOCK_ALLIDS( *idl ) ) {
 837                 return( 2 );    /* already there */
 838         }
 839
 840         /* is it already there? */
 841         i = idl_find(*idl, id);
 842         if ( ID_BLOCK_ID(*idl, i) == id ) {
 843                 return( 2 );    /* already there */
 844         }
 845         if ( ID_BLOCK_NIDS(*idl) && ID_BLOCK_ID(*idl, i) < id )
 846                 i++;
 847
 848         /* do we need to make room for it? */
 849         if ( ID_BLOCK_NIDS(*idl) == ID_BLOCK_NMAXN(*idl) ) {
 850                 /* make room or indicate block needs splitting */
 851                 if ( ID_BLOCK_NMAXN(*idl) >= maxids ) {
 852                         return( 3 );    /* block needs splitting */
 853                 }
 854
 855                 ID_BLOCK_NMAX(*idl) *= 2;
 856                 if ( ID_BLOCK_NMAXN(*idl) > maxids ) {
 857                         ID_BLOCK_NMAX(*idl) = maxids;
 858                 }
 859                 *idl = (ID_BLOCK *) ch_realloc( (char *) *idl,
 860                     (ID_BLOCK_NMAXN(*idl) + ID_BLOCK_IDS_OFFSET) * sizeof(ID) );
 861         }
 862
 863         /* make a slot for the new id */
 864         AC_MEMCPY( &ID_BLOCK_ID(*idl, i+1), &ID_BLOCK_ID(*idl, i),
 865                     (ID_BLOCK_NIDS(*idl) - i) * sizeof(ID) );
 866
 867         ID_BLOCK_ID(*idl, i) = id;
 868         ID_BLOCK_NIDS(*idl)++;
 869         (void) memset(
 870                 (char *) &ID_BLOCK_ID((*idl), ID_BLOCK_NIDS(*idl)),
 871                 '\0',
 872             (ID_BLOCK_NMAXN(*idl) - ID_BLOCK_NIDS(*idl)) * sizeof(ID) );
 873
 874 #ifdef LDBM_DEBUG_IDL
 875         idl_check(*idl);
 876 #endif
 877
 878         return( i == 0 ? 1 : 0 );       /* inserted - first id changed or not */
 879 }
 880
 881
 882 int
 883 idl_delete_key (
 884         Backend         *be,
 885         DBCache  *db,
 886         Datum           key,
 887         ID              id
 888 )
 889 {
 890         Datum  data;
 891         ID_BLOCK *idl;
 892         unsigned i;
 893         int j, nids;
 894
 895         if ( (idl = idl_fetch_one( be, db, key ) ) == NULL )
 896         {
 897                 /* It wasn't found.  Hmm... */
 898                 return -1;
 899         }
 900
 901         if ( ID_BLOCK_ALLIDS( idl ) ) {
 902                 idl_free( idl );
 903                 return 0;
 904         }
 905
 906         if ( ! ID_BLOCK_INDIRECT( idl ) ) {
 907                 i = idl_find(idl, id);
 908                 if ( ID_BLOCK_ID(idl, i) == id ) {
 909                         if( --ID_BLOCK_NIDS(idl) == 0 ) {
 910                                 ldbm_cache_delete( db, key );
 911
 912                         } else {
 913                                 AC_MEMCPY(
 914                                         &ID_BLOCK_ID(idl, i),
 915                                         &ID_BLOCK_ID(idl, i+1),
 916                                         (ID_BLOCK_NIDS(idl)-i) * sizeof(ID) );
 917
 918                                 ID_BLOCK_ID(idl, ID_BLOCK_NIDS(idl)) = NOID;
 919
 920                                 idl_store( be, db, key, idl );
 921                         }
 922
 923                         idl_free( idl );
 924                         return 0;
 925                 }
 926                 /*  We didn't find the ID.  Hmmm... */
 927                 idl_free( idl );
 928                 return -1;
 929         }
 930
 931         /* We have to go through an indirect block and find the ID
 932            in the list of IDL's
 933            */
 934         cont_alloc( &data, &key );
 935 #ifndef USE_INDIRECT_NIDS
 936         for ( nids = 0; !ID_BLOCK_NOID(idl, nids); nids++ )
 937                 ;       /* NULL */
 938
 939         for ( j = 0; j<nids; j++ )
 940 #else
 941         nids = ID_BLOCK_NIDS(idl);
 942         for ( j = idl_find(idl, id); j >= 0; j = -1)    /* execute once */
 943 #endif
 944         {
 945                 ID_BLOCK *tmp;
 946                 cont_id( &data, ID_BLOCK_ID(idl, j) );
 947
 948                 if ( (tmp = idl_fetch_one( be, db, data )) == NULL ) {
 949 #ifdef NEW_LOGGING
 950                         LDAP_LOG( INDEX, INFO,
 951                                    "idl_delete_key: idl_fetch_one returned NULL\n", 0, 0, 0 );
 952 #else
 953                         Debug( LDAP_DEBUG_ANY,
 954                             "idl_delete_key: idl_fetch of returned NULL\n", 0, 0, 0 );
 955 #endif
 956
 957                         continue;
 958                 }
 959                 /*
 960                    Now try to find the ID in tmp
 961                 */
 962
 963                 i = idl_find(tmp, id);
 964                 if ( ID_BLOCK_ID(tmp, i) == id )
 965                 {
 966                         AC_MEMCPY(
 967                                 &ID_BLOCK_ID(tmp, i),
 968                                 &ID_BLOCK_ID(tmp, i+1),
 969                                 (ID_BLOCK_NIDS(tmp)-(i+1)) * sizeof(ID));
 970                         ID_BLOCK_ID(tmp, ID_BLOCK_NIDS(tmp)-1 ) = NOID;
 971                         ID_BLOCK_NIDS(tmp)--;
 972
 973                         if ( ID_BLOCK_NIDS(tmp) ) {
 974                                 idl_store ( be, db, data, tmp );
 975
 976                         } else {
 977                                 ldbm_cache_delete( db, data );
 978                                 AC_MEMCPY(
 979                                         &ID_BLOCK_ID(idl, j),
 980                                         &ID_BLOCK_ID(idl, j+1),
 981                                         (nids-(j+1)) * sizeof(ID));
 982                                 ID_BLOCK_ID(idl, nids-1) = NOID;
 983                                 nids--;
 984 #ifdef USE_INDIRECT_NIDS
 985                                 ID_BLOCK_NIDS(idl)--;
 986 #endif
 987                                 if ( ! nids )
 988                                         ldbm_cache_delete( db, key );
 989                                 else
 990                                         idl_store( be, db, key, idl );
 991                         }
 992                         idl_free( tmp );
 993                         cont_free( &data );
 994                         idl_free( idl );
 995                         return 0;
 996                 }
 997                 idl_free( tmp );
 998         }
 999
1000         cont_free( &data );
1001         idl_free( idl );
1002         return -1;
1003 }
1004
1005
1006 /* return a duplicate of a single ID_BLOCK */
1007 static ID_BLOCK *
1008 idl_dup( ID_BLOCK *idl )
1009 {
1010         ID_BLOCK        *new;
1011
1012         if ( idl == NULL ) {
1013                 return( NULL );
1014         }
1015
1016         new = idl_alloc( ID_BLOCK_NMAXN(idl) );
1017
1018         AC_MEMCPY(
1019                 (char *) new,
1020                 (char *) idl,
1021                 (ID_BLOCK_NMAXN(idl) + ID_BLOCK_IDS_OFFSET) * sizeof(ID) );
1022
1023 #ifdef LDBM_DEBUG_IDL
1024         idl_check(new);
1025 #endif
1026
1027         return( new );
1028 }
1029
1030
1031 /* return the smaller ID_BLOCK */
1032 static ID_BLOCK *
1033 idl_min( ID_BLOCK *a, ID_BLOCK *b )
1034 {
1035         return( ID_BLOCK_NIDS(a) > ID_BLOCK_NIDS(b) ? b : a );
1036 }
1037
1038
1039 /*
1040  * idl_intersection - return a intersection b
1041  */
1042 ID_BLOCK *
1043 idl_intersection(
1044     Backend     *be,
1045     ID_BLOCK    *a,
1046     ID_BLOCK    *b
1047 )
1048 {
1049         unsigned int    ai, bi, ni;
1050         ID_BLOCK                *n;
1051
1052         if ( a == NULL || b == NULL ) {
1053                 return( NULL );
1054         }
1055         if ( ID_BLOCK_ALLIDS( a ) ) {
1056                 return( idl_dup( b ) );
1057         }
1058         if ( ID_BLOCK_ALLIDS( b ) ) {
1059                 return( idl_dup( a ) );
1060         }
1061         if ( ID_BLOCK_NIDS(a) == 0 || ID_BLOCK_NIDS(b) == 0 ) {
1062                 return( NULL );
1063         }
1064
1065         n = idl_dup( idl_min( a, b ) );
1066
1067 #ifdef LDBM_DEBUG_IDL
1068         idl_check(a);
1069         idl_check(b);
1070 #endif
1071
1072         for ( ni = 0, ai = 0, bi = 0; ; ) {
1073                 if ( ID_BLOCK_ID(b, bi) == ID_BLOCK_ID(a, ai) ) {
1074                         ID_BLOCK_ID(n, ni++) = ID_BLOCK_ID(a, ai);
1075                         ai++;
1076                         bi++;
1077                         if ( ai >= ID_BLOCK_NIDS(a) || bi >= ID_BLOCK_NIDS(b) )
1078                                 break;
1079                 } else if ( ID_BLOCK_ID(a, ai) < ID_BLOCK_ID(b, bi) ) {
1080                         ai++;
1081                         if ( ai >= ID_BLOCK_NIDS(a) )
1082                                 break;
1083                 } else {
1084                         bi++;
1085                         if ( bi >= ID_BLOCK_NIDS(b) )
1086                                 break;
1087                 }
1088         }
1089
1090         if ( ni == 0 ) {
1091                 idl_free( n );
1092                 return( NULL );
1093         }
1094         ID_BLOCK_NIDS(n) = ni;
1095
1096 #ifdef LDBM_DEBUG_IDL
1097         idl_check(n);
1098 #endif
1099
1100         return( n );
1101 }
1102
1103
1104 /*
1105  * idl_union - return a union b
1106  */
1107 ID_BLOCK *
1108 idl_union(
1109     Backend     *be,
1110     ID_BLOCK    *a,
1111     ID_BLOCK    *b
1112 )
1113 {
1114         unsigned int    ai, bi, ni;
1115         ID_BLOCK                *n;
1116
1117         if ( a == NULL ) {
1118                 return( idl_dup( b ) );
1119         }
1120         if ( b == NULL ) {
1121                 return( idl_dup( a ) );
1122         }
1123         if ( ID_BLOCK_ALLIDS( a ) || ID_BLOCK_ALLIDS( b ) ) {
1124                 return( idl_allids( be ) );
1125         }
1126
1127 #ifdef LDBM_DEBUG_IDL
1128         idl_check(a);
1129         idl_check(b);
1130 #endif
1131
1132         if ( ID_BLOCK_NIDS(b) < ID_BLOCK_NIDS(a) ) {
1133                 n = a;
1134                 a = b;
1135                 b = n;
1136         }
1137
1138         n = idl_alloc( ID_BLOCK_NIDS(a) + ID_BLOCK_NIDS(b) );
1139
1140         for ( ni = 0, ai = 0, bi = 0;
1141                 ai < ID_BLOCK_NIDS(a) && bi < ID_BLOCK_NIDS(b);
1142                 )
1143         {
1144                 if ( ID_BLOCK_ID(a, ai) < ID_BLOCK_ID(b, bi) ) {
1145                         ID_BLOCK_ID(n, ni++) = ID_BLOCK_ID(a, ai++);
1146
1147                 } else if ( ID_BLOCK_ID(b, bi) < ID_BLOCK_ID(a, ai) ) {
1148                         ID_BLOCK_ID(n, ni++) = ID_BLOCK_ID(b, bi++);
1149
1150                 } else {
1151                         ID_BLOCK_ID(n, ni++) = ID_BLOCK_ID(a, ai);
1152                         ai++, bi++;
1153                 }
1154         }
1155
1156         for ( ; ai < ID_BLOCK_NIDS(a); ai++ ) {
1157                 ID_BLOCK_ID(n, ni++) = ID_BLOCK_ID(a, ai);
1158         }
1159         for ( ; bi < ID_BLOCK_NIDS(b); bi++ ) {
1160                 ID_BLOCK_ID(n, ni++) = ID_BLOCK_ID(b, bi);
1161         }
1162         ID_BLOCK_NIDS(n) = ni;
1163
1164 #ifdef LDBM_DEBUG_IDL
1165         idl_check(n);
1166 #endif
1167
1168         return( n );
1169 }
1170
1171
1172 /*
1173  * idl_notin - return a intersection ~b (or a minus b)
1174  */
1175 ID_BLOCK *
1176 idl_notin(
1177     Backend     *be,
1178     ID_BLOCK    *a,
1179     ID_BLOCK    *b
1180 )
1181 {
1182         unsigned int    ni, ai, bi;
1183         ID_BLOCK                *n;
1184
1185         if ( a == NULL ) {
1186                 return( NULL );
1187         }
1188         if ( b == NULL || ID_BLOCK_ALLIDS( b )) {
1189                 return( idl_dup( a ) );
1190         }
1191
1192         if ( ID_BLOCK_ALLIDS( a ) ) {
1193                 n = idl_alloc( SLAPD_LDBM_MIN_MAXIDS );
1194                 ni = 0;
1195
1196                 for ( ai = 1, bi = 0;
1197                         ai < ID_BLOCK_NIDS(a) && ni < ID_BLOCK_NMAXN(n) && bi < ID_BLOCK_NMAXN(b);
1198                         ai++ )
1199                 {
1200                         if ( ID_BLOCK_ID(b, bi) == ai ) {
1201                                 bi++;
1202                         } else {
1203                                 ID_BLOCK_ID(n, ni++) = ai;
1204                         }
1205                 }
1206
1207                 for ( ; ai < ID_BLOCK_NIDS(a) && ni < ID_BLOCK_NMAXN(n); ai++ ) {
1208                         ID_BLOCK_ID(n, ni++) = ai;
1209                 }
1210
1211                 if ( ni == ID_BLOCK_NMAXN(n) ) {
1212                         idl_free( n );
1213                         return( idl_allids( be ) );
1214                 } else {
1215                         ID_BLOCK_NIDS(n) = ni;
1216                         return( n );
1217                 }
1218         }
1219
1220         n = idl_dup( a );
1221
1222         ni = 0;
1223         for ( ai = 0, bi = 0; ai < ID_BLOCK_NIDS(a); ai++ ) {
1224                 for ( ;
1225                         bi < ID_BLOCK_NIDS(b) && ID_BLOCK_ID(b, bi) < ID_BLOCK_ID(a, ai);
1226                     bi++ )
1227                 {
1228                         ;       /* NULL */
1229                 }
1230
1231                 if ( bi == ID_BLOCK_NIDS(b) ) {
1232                         break;
1233                 }
1234
1235                 if ( ID_BLOCK_ID(b, bi) != ID_BLOCK_ID(a, ai) ) {
1236                         ID_BLOCK_ID(n, ni++) = ID_BLOCK_ID(a, ai);
1237                 }
1238         }
1239
1240         for ( ; ai < ID_BLOCK_NIDS(a); ai++ ) {
1241                 ID_BLOCK_ID(n, ni++) = ID_BLOCK_ID(a, ai);
1242         }
1243         ID_BLOCK_NIDS(n) = ni;
1244
1245 #ifdef LDBM_DEBUG_IDL
1246         idl_check(n);
1247 #endif
1248
1249         return( n );
1250 }
1251
1252 /*      return the first ID in the block
1253  *      if ALLIDS block
1254  *              NIDS > 1 return 1
1255  *              otherwise return NOID
1256  *      otherwise return first ID
1257  *
1258  *      cursor is set to 1
1259  */
1260 ID
1261 idl_firstid( ID_BLOCK *idl, ID *cursor )
1262 {
1263         *cursor = 1;
1264
1265         if ( idl == NULL || ID_BLOCK_NIDS(idl) == 0 ) {
1266                 return( NOID );
1267         }
1268
1269         if ( ID_BLOCK_ALLIDS( idl ) ) {
1270                 return( ID_BLOCK_NIDS(idl) > 1 ? 1 : NOID );
1271         }
1272
1273         return( ID_BLOCK_ID(idl, 0) );
1274 }
1275
1276 /*      return next ID
1277  *      if ALLIDS block, cursor is id.
1278  *              increment id
1279  *              if id < NIDS return id
1280  *              otherwise NOID.
1281  *      otherwise cursor is index into block
1282  *              if index < nids
1283  *                      return id at index then increment
1284  */
1285 ID
1286 idl_nextid( ID_BLOCK *idl, ID *cursor )
1287 {
1288         if ( ID_BLOCK_ALLIDS( idl ) ) {
1289                 if( ++(*cursor) < ID_BLOCK_NIDS(idl) ) {
1290                         return *cursor;
1291                 } else {
1292                         return NOID;
1293                 }
1294         }
1295
1296         if ( *cursor < ID_BLOCK_NIDS(idl) ) {
1297                 return( ID_BLOCK_ID(idl, (*cursor)++) );
1298         }
1299
1300         return( NOID );
1301 }