git.sur5r.net Git - openldap/blob - servers/slapd/back-ldbm/idl.c

   1 /* idl.c - ldap id list handling routines */
   2 /* $OpenLDAP$ */
   3 /*
   4  * Copyright 1998-2000 The OpenLDAP Foundation, All Rights Reserved.
   5  * COPYING RESTRICTIONS APPLY, see COPYRIGHT file
   6  */
   7
   8 #include "portable.h"
   9
  10 #include <stdio.h>
  11
  12 #include <ac/string.h>
  13 #include <ac/socket.h>
  14
  15 #include "slap.h"
  16 #include "back-ldbm.h"
  17
  18 static ID_BLOCK* idl_dup( ID_BLOCK *idl );
  19
  20 static void cont_alloc( Datum *cont, Datum *key )
  21 {
  22         ldbm_datum_init( *cont );
  23         cont->dsize = 1 + sizeof(ID) + key->dsize;
  24         cont->dptr = ch_malloc( cont->dsize );
  25
  26         * (unsigned char *) cont->dptr = SLAP_INDEX_CONT_PREFIX;
  27
  28         AC_MEMCPY( &((unsigned char *)cont->dptr)[1 + sizeof(ID)],
  29                 key->dptr, key->dsize );
  30 }
  31
  32 static void cont_id( Datum *cont, ID id )
  33 {
  34         int i;
  35
  36         for( i=1; i <= sizeof(id); i++) {
  37                 ((unsigned char *)cont->dptr)[i] = (unsigned char)(id & 0xFF);
  38                 id >>= 8;
  39         }
  40
  41 }
  42
  43 static void cont_free( Datum *cont )
  44 {
  45         ch_free( cont->dptr );
  46 }
  47
  48 #ifdef LDBM_DEBUG_IDL
  49 static void idl_check(ID_BLOCK *idl)
  50 {
  51         int i;
  52         ID_BLOCK last;
  53
  54         if( ID_BLOCK_INDIRECT(idl) || ID_BLOCK_ALLIDS(idl)
  55                 || ID_BLOCK_NIDS(idl) <= 1 )
  56         {
  57                 return;
  58         }
  59
  60         for( last = ID_BLOCK_ID(idl, 0), i = 1;
  61                 i < ID_BLOCK_NIDS(idl);
  62                 last = ID_BLOCK_ID(idl, i), i++ )
  63         {
  64                 assert (last < ID_BLOCK_ID(idl, i) );
  65         }
  66 }
  67 #endif
  68
  69 /* Allocate an ID_BLOCK with room for nids ids */
  70 ID_BLOCK *
  71 idl_alloc( unsigned int nids )
  72 {
  73         ID_BLOCK        *new;
  74
  75         /* nmax + nids + space for the ids */
  76         new = (ID_BLOCK *) ch_calloc( (ID_BLOCK_IDS_OFFSET + nids), sizeof(ID) );
  77         ID_BLOCK_NMAX(new) = nids;
  78         ID_BLOCK_NIDS(new) = 0;
  79
  80         return( new );
  81 }
  82
  83
  84 /* Allocate an empty ALLIDS ID_BLOCK */
  85 ID_BLOCK        *
  86 idl_allids( Backend *be )
  87 {
  88         ID_BLOCK        *idl;
  89         ID              id;
  90
  91         idl = idl_alloc( 0 );
  92         ID_BLOCK_NMAX(idl) = ID_BLOCK_ALLIDS_VALUE;
  93         if ( next_id_get( be, &id ) ) {
  94                 return NULL;
  95         }
  96         ID_BLOCK_NIDS(idl) = id;
  97
  98         return( idl );
  99 }
 100
 101 /* Free an ID_BLOCK */
 102 void
 103 idl_free( ID_BLOCK *idl )
 104 {
 105         if ( idl == NULL ) {
 106 #ifdef NEW_LOGGING
 107                 LDAP_LOG(( "cache", LDAP_LEVEL_INFO,
 108                            "idl_free: called with NULL pointer\n" ));
 109 #else
 110                 Debug( LDAP_DEBUG_TRACE,
 111                         "idl_free: called with NULL pointer\n",
 112                         0, 0, 0 );
 113 #endif
 114
 115                 return;
 116         }
 117
 118         free( (char *) idl );
 119 }
 120
 121
 122 /* Fetch an single ID_BLOCK from the cache */
 123 static ID_BLOCK *
 124 idl_fetch_one(
 125     Backend             *be,
 126     DBCache     *db,
 127     Datum               key
 128 )
 129 {
 130         Datum   data;
 131         ID_BLOCK        *idl;
 132
 133         /* Debug( LDAP_DEBUG_TRACE, "=> idl_fetch_one\n", 0, 0, 0 ); */
 134
 135         data = ldbm_cache_fetch( db, key );
 136
 137         if( data.dptr == NULL ) {
 138                 return NULL;
 139         }
 140
 141         idl = idl_dup((ID_BLOCK *) data.dptr);
 142
 143         ldbm_datum_free( db->dbc_db, data );
 144
 145         return idl;
 146 }
 147
 148
 149 /* Fetch a set of ID_BLOCKs from the cache
 150  *      if not INDIRECT
 151  *              if block return is an ALLIDS block,
 152  *                      return an new ALLIDS block
 153  *              otherwise
 154  *                      return block
 155  *      construct super block from all blocks referenced by INDIRECT block
 156  *      return super block
 157  */
 158 ID_BLOCK *
 159 idl_fetch(
 160     Backend             *be,
 161     DBCache     *db,
 162     Datum               key
 163 )
 164 {
 165         Datum   data;
 166         ID_BLOCK        *idl;
 167         ID_BLOCK        **tmp;
 168         int     i, nids;
 169
 170         idl = idl_fetch_one( be, db, key );
 171
 172         if ( idl == NULL ) {
 173                 return NULL;
 174         }
 175
 176         if ( ID_BLOCK_ALLIDS(idl) ) {
 177                 /* all ids block */
 178                 /* make sure we have the current value of highest id */
 179                 idl_free( idl );
 180                 idl = idl_allids( be );
 181
 182                 return( idl );
 183         }
 184
 185         if ( ! ID_BLOCK_INDIRECT( idl ) ) {
 186                 /* regular block */
 187                 return( idl );
 188         }
 189
 190         /*
 191          * this is an indirect block which points to other blocks.
 192          * we need to read in all the blocks it points to and construct
 193          * a big id list containing all the ids, which we will return.
 194          */
 195
 196         /* count the number of blocks & allocate space for pointers to them */
 197         for ( i = 0; !ID_BLOCK_NOID(idl, i); i++ )
 198                 ;       /* NULL */
 199         tmp = (ID_BLOCK **) ch_malloc( (i + 1) * sizeof(ID_BLOCK *) );
 200
 201         /* read in all the blocks */
 202         cont_alloc( &data, &key );
 203         nids = 0;
 204         for ( i = 0; !ID_BLOCK_NOID(idl, i); i++ ) {
 205                 cont_id( &data, ID_BLOCK_ID(idl, i) );
 206
 207                 if ( (tmp[i] = idl_fetch_one( be, db, data )) == NULL ) {
 208 #ifdef NEW_LOGGING
 209                         LDAP_LOG(( "cache", LDAP_LEVEL_INFO,
 210                                    "idl_fetch: idl_fetch_one returned NULL\n" ));
 211 #else
 212                         Debug( LDAP_DEBUG_ANY,
 213                             "idl_fetch: one returned NULL\n", 0, 0, 0 );
 214 #endif
 215
 216                         continue;
 217                 }
 218
 219                 nids += ID_BLOCK_NIDS(tmp[i]);
 220         }
 221         tmp[i] = NULL;
 222         cont_free( &data );
 223         idl_free( idl );
 224
 225         /* allocate space for the big block */
 226         idl = idl_alloc( nids );
 227         ID_BLOCK_NIDS(idl) = nids;
 228         nids = 0;
 229
 230         /* copy in all the ids from the component blocks */
 231         for ( i = 0; tmp[i] != NULL; i++ ) {
 232                 if ( tmp[i] == NULL ) {
 233                         continue;
 234                 }
 235
 236                 AC_MEMCPY(
 237                         (char *) &ID_BLOCK_ID(idl, nids),
 238                         (char *) &ID_BLOCK_ID(tmp[i], 0),
 239                         ID_BLOCK_NIDS(tmp[i]) * sizeof(ID) );
 240                 nids += ID_BLOCK_NIDS(tmp[i]);
 241
 242                 idl_free( tmp[i] );
 243         }
 244         free( (char *) tmp );
 245
 246 #ifdef LDBM_DEBUG_IDL
 247         idl_check(idl);
 248 #endif
 249
 250 #ifdef NEW_LOGGING
 251         LDAP_LOG(( "cache", LDAP_LEVEL_ENTRY,
 252                    "idl_fetch: %ld ids (%ld max)\n",
 253                    ID_BLOCK_NIDS(idl), ID_BLOCK_NMAX(idl) ));
 254 #else
 255         Debug( LDAP_DEBUG_TRACE, "<= idl_fetch %ld ids (%ld max)\n",
 256                ID_BLOCK_NIDS(idl), ID_BLOCK_NMAX(idl), 0 );
 257 #endif
 258
 259         return( idl );
 260 }
 261
 262
 263 /* store a single block */
 264 static int
 265 idl_store(
 266     Backend             *be,
 267     DBCache     *db,
 268     Datum               key,
 269     ID_BLOCK            *idl
 270 )
 271 {
 272         int     rc, flags;
 273         Datum   data;
 274         struct ldbminfo *li = (struct ldbminfo *) be->be_private;
 275
 276 #ifdef LDBM_DEBUG_IDL
 277         idl_check(idl);
 278 #endif
 279
 280         ldbm_datum_init( data );
 281
 282         /* Debug( LDAP_DEBUG_TRACE, "=> idl_store\n", 0, 0, 0 ); */
 283
 284         data.dptr = (char *) idl;
 285         data.dsize = (ID_BLOCK_IDS_OFFSET + ID_BLOCK_NMAX(idl)) * sizeof(ID);
 286
 287 #ifdef LDBM_DEBUG
 288         Statslog( LDAP_DEBUG_STATS, "<= idl_store(): rc=%d\n",
 289                 rc, 0, 0, 0, 0 );
 290 #endif
 291
 292         flags = LDBM_REPLACE;
 293         rc = ldbm_cache_store( db, key, data, flags );
 294
 295         /* Debug( LDAP_DEBUG_TRACE, "<= idl_store %d\n", rc, 0, 0 ); */
 296         return( rc );
 297 }
 298
 299 /* Binary search for id in block, return index
 300  *    an index is always returned, even with no match. If no
 301  * match, the returned index is the insertion point.
 302  */
 303 static unsigned int
 304 idl_find(
 305     ID_BLOCK    *b,
 306     ID          id
 307 )
 308 {
 309         int lo=0, hi=ID_BLOCK_NIDS(b)-1, nr;
 310
 311         for (;lo<=hi;)
 312         {
 313             nr = ( lo + hi ) / 2;
 314             if (ID_BLOCK_ID(b, nr) == id)
 315                 break;
 316             if (ID_BLOCK_ID(b, nr) > id)
 317                 hi = nr - 1;
 318             else
 319                 lo = nr + 1;
 320         }
 321         return nr;
 322 }
 323
 324 /* split the block at id
 325  *      locate ID greater than or equal to id.
 326  */
 327 static void
 328 idl_split_block(
 329     ID_BLOCK    *b,
 330     ID          id,
 331     ID_BLOCK    **right,
 332     ID_BLOCK    **left
 333 )
 334 {
 335         unsigned int    nr, nl;
 336
 337         /* find where to split the block */
 338         nr = idl_find(b, id);
 339         if ( ID_BLOCK_ID(b,nr) < id )
 340                 nr++;
 341
 342         nl = ID_BLOCK_NIDS(b) - nr;
 343
 344         *right = idl_alloc( nr == 0 ? 1 : nr );
 345         *left = idl_alloc( nl + (nr == 0 ? 0 : 1));
 346
 347         /*
 348          * everything before the id being inserted in the first block
 349          * unless there is nothing, in which case the id being inserted
 350          * goes there.
 351          */
 352         if ( nr == 0 ) {
 353                 ID_BLOCK_NIDS(*right) = 1;
 354                 ID_BLOCK_ID(*right, 0) = id;
 355         } else {
 356                 AC_MEMCPY(
 357                         (char *) &ID_BLOCK_ID(*right, 0),
 358                         (char *) &ID_BLOCK_ID(b, 0),
 359                         nr * sizeof(ID) );
 360                 ID_BLOCK_NIDS(*right) = nr;
 361                 ID_BLOCK_ID(*left, 0) = id;
 362         }
 363
 364         /* the id being inserted & everything after in the second block */
 365         AC_MEMCPY(
 366                 (char *) &ID_BLOCK_ID(*left, (nr == 0 ? 0 : 1)),
 367             (char *) &ID_BLOCK_ID(b, nr),
 368                 nl * sizeof(ID) );
 369         ID_BLOCK_NIDS(*left) = nl + (nr == 0 ? 0 : 1);
 370
 371 #ifdef LDBM_DEBUG_IDL
 372         idl_check(*right);
 373         idl_check(*left);
 374 #endif
 375 }
 376
 377
 378 /*
 379  * idl_change_first - called when an indirect block's first key has
 380  * changed, meaning it needs to be stored under a new key, and the
 381  * header block pointing to it needs updating.
 382  */
 383 static int
 384 idl_change_first(
 385     Backend             *be,
 386     DBCache     *db,
 387     Datum               hkey,           /* header block key     */
 388     ID_BLOCK            *h,             /* header block         */
 389     int                 pos,            /* pos in h to update   */
 390     Datum               bkey,           /* data block key       */
 391     ID_BLOCK            *b              /* data block           */
 392 )
 393 {
 394         int     rc;
 395
 396         /* Debug( LDAP_DEBUG_TRACE, "=> idl_change_first\n", 0, 0, 0 ); */
 397
 398         /* delete old key block */
 399         if ( (rc = ldbm_cache_delete( db, bkey )) != 0 ) {
 400 #ifdef NEW_LOGGING
 401                 LDAP_LOG(( "cache", LDAP_LEVEL_INFO,
 402                            "idl_change_first: ldbm_cache_delete returned %d\n", rc ));
 403 #else
 404                 Debug( LDAP_DEBUG_ANY,
 405                     "idl_change_first: ldbm_cache_delete returned %d\n",
 406                         rc, 0, 0 );
 407 #endif
 408
 409                 return( rc );
 410         }
 411
 412         /* write block with new key */
 413         cont_id( &bkey, ID_BLOCK_ID(b, 0) );
 414
 415         if ( (rc = idl_store( be, db, bkey, b )) != 0 ) {
 416 #ifdef NEW_LOGGING
 417                 LDAP_LOG(( "cache", LDAP_LEVEL_INFO,
 418                            "idl_change_first: idl_store returned %d\n", rc ));
 419 #else
 420                 Debug( LDAP_DEBUG_ANY,
 421                     "idl_change_first: idl_store returned %d\n", rc, 0, 0 );
 422 #endif
 423
 424                 return( rc );
 425         }
 426
 427         /* update + write indirect header block */
 428         ID_BLOCK_ID(h, pos) = ID_BLOCK_ID(b, 0);
 429         if ( (rc = idl_store( be, db, hkey, h )) != 0 ) {
 430 #ifdef NEW_LOGGING
 431                 LDAP_LOG(( "cache", LDAP_LEVEL_INFO,
 432                            "idl_change_first: idl_store returned %s\n", rc ));
 433 #else
 434                 Debug( LDAP_DEBUG_ANY,
 435                     "idl_change_first: idl_store returned %d\n", rc, 0, 0 );
 436 #endif
 437
 438                 return( rc );
 439         }
 440
 441         return( 0 );
 442 }
 443
 444
 445 int
 446 idl_insert_key(
 447     Backend             *be,
 448     DBCache     *db,
 449     Datum               key,
 450     ID                  id
 451 )
 452 {
 453         int     i, j, first, rc;
 454         ID_BLOCK        *idl, *tmp, *tmp2, *tmp3;
 455         Datum   k2;
 456
 457         if ( (idl = idl_fetch_one( be, db, key )) == NULL ) {
 458                 idl = idl_alloc( 1 );
 459                 ID_BLOCK_ID(idl, ID_BLOCK_NIDS(idl)++) = id;
 460                 rc = idl_store( be, db, key, idl );
 461
 462                 idl_free( idl );
 463                 return( rc );
 464         }
 465
 466         if ( ID_BLOCK_ALLIDS( idl ) ) {
 467                 /* ALLIDS */
 468                 idl_free( idl );
 469                 return 0;
 470         }
 471
 472         if ( ! ID_BLOCK_INDIRECT( idl ) ) {
 473                 /* regular block */
 474                 switch ( idl_insert( &idl, id, db->dbc_maxids ) ) {
 475                 case 0:         /* id inserted - store the updated block */
 476                 case 1:
 477                         rc = idl_store( be, db, key, idl );
 478                         break;
 479
 480                 case 2:         /* id already there - nothing to do */
 481                         rc = 0;
 482                         break;
 483
 484                 case 3:         /* id not inserted - block must be split */
 485                         /* check threshold for marking this an all-id block */
 486                         if ( db->dbc_maxindirect < 2 ) {
 487                                 idl_free( idl );
 488                                 idl = idl_allids( be );
 489                                 rc = idl_store( be, db, key, idl );
 490                                 break;
 491                         }
 492
 493                         idl_split_block( idl, id, &tmp, &tmp2 );
 494                         idl_free( idl );
 495
 496                         /* create the header indirect block */
 497                         idl = idl_alloc( 3 );
 498                         ID_BLOCK_NMAX(idl) = 3;
 499                         ID_BLOCK_NIDS(idl) = ID_BLOCK_INDIRECT_VALUE;
 500                         ID_BLOCK_ID(idl, 0) = ID_BLOCK_ID(tmp, 0);
 501                         ID_BLOCK_ID(idl, 1) = ID_BLOCK_ID(tmp2, 0);
 502                         ID_BLOCK_ID(idl, 2) = NOID;
 503
 504                         /* store it */
 505                         rc = idl_store( be, db, key, idl );
 506
 507                         cont_alloc( &k2, &key );
 508                         cont_id( &k2, ID_BLOCK_ID(tmp, 0) );
 509
 510                         rc = idl_store( be, db, k2, tmp );
 511
 512                         cont_id( &k2, ID_BLOCK_ID(tmp2, 0) );
 513                         rc = idl_store( be, db, k2, tmp2 );
 514
 515                         cont_free( &k2 );
 516
 517                         idl_free( tmp );
 518                         idl_free( tmp2 );
 519                         break;
 520                 }
 521
 522                 idl_free( idl );
 523                 return( rc );
 524         }
 525
 526         /*
 527          * this is an indirect block which points to other blocks.
 528          * we need to read in the block into which the id should be
 529          * inserted, then insert the id and store the block.  we might
 530          * have to split the block if it is full, which means we also
 531          * need to write a new "header" block.
 532          */
 533
 534         /* select the block to try inserting into *//* XXX linear search XXX */
 535         for ( i = 0; !ID_BLOCK_NOID(idl, i) && id > ID_BLOCK_ID(idl, i); i++ )
 536                 ;       /* NULL */
 537
 538         if ( i != 0 ) {
 539                 i--;
 540                 first = 0;
 541         } else {
 542                 first = 1;
 543         }
 544
 545         /* get the block */
 546         cont_alloc( &k2, &key );
 547         cont_id( &k2, ID_BLOCK_ID(idl, i) );
 548
 549         if ( (tmp = idl_fetch_one( be, db, k2 )) == NULL ) {
 550 #ifdef NEW_LOGGING
 551                 LDAP_LOG(( "cache", LDAP_LEVEL_ERR,
 552                            "idl_insert_key: nonexistent continuation block\n" ));
 553 #else
 554                 Debug( LDAP_DEBUG_ANY, "idl_insert_key: nonexistent continuation block\n",
 555                     0, 0, 0 );
 556 #endif
 557
 558                 cont_free( &k2 );
 559                 idl_free( idl );
 560                 return( -1 );
 561         }
 562
 563         /* insert the id */
 564         switch ( idl_insert( &tmp, id, db->dbc_maxids ) ) {
 565         case 0:         /* id inserted ok */
 566                 if ( (rc = idl_store( be, db, k2, tmp )) != 0 ) {
 567 #ifdef NEW_LOGGING
 568                         LDAP_LOG(( "cache", LDAP_LEVEL_ERR,
 569                                    "ids_insert_key: idl_store returned %d\n", rc ));
 570 #else
 571                         Debug( LDAP_DEBUG_ANY,
 572                             "idl_insert_key: idl_store returned %d\n", rc, 0, 0 );
 573 #endif
 574
 575                 }
 576                 break;
 577
 578         case 1:         /* id inserted - first id in block has changed */
 579                 /*
 580                  * key for this block has changed, so we have to
 581                  * write the block under the new key, delete the
 582                  * old key block + update and write the indirect
 583                  * header block.
 584                  */
 585
 586                 rc = idl_change_first( be, db, key, idl, i, k2, tmp );
 587                 break;
 588
 589         case 2:         /* id not inserted - already there, do nothing */
 590                 rc = 0;
 591                 break;
 592
 593         case 3:         /* id not inserted - block is full */
 594                 /*
 595                  * first, see if it will fit in the next block,
 596                  * without splitting, unless we're trying to insert
 597                  * into the beginning of the first block.
 598                  */
 599
 600                 /* is there a next block? */
 601                 if ( !first && !ID_BLOCK_NOID(idl, i + 1) ) {
 602                         /* read it in */
 603                         cont_alloc( &k2, &key );
 604                         cont_id( &k2, ID_BLOCK_ID(idl, i) );
 605                         if ( (tmp2 = idl_fetch_one( be, db, k2 )) == NULL ) {
 606 #ifdef NEW_LOGGING
 607                                 LDAP_LOG(( "cache", LDAP_LEVEL_ERR,
 608                                            "idl_insert_key: idl_fetch_one returned NULL\n"));
 609 #else
 610                                 Debug( LDAP_DEBUG_ANY,
 611                                     "idl_insert_key: idl_fetch_one returned NULL\n",
 612                                     0, 0, 0 );
 613 #endif
 614
 615                                 /* split the original block */
 616                                 cont_free( &k2 );
 617                                 goto split;
 618                         }
 619
 620                         /* If the new id is less than the last id in the
 621                          * current block, it must not be put into the next
 622                          * block. Push the last id of the current block
 623                          * into the next block instead.
 624                          */
 625                         if (id < ID_BLOCK_ID(tmp, ID_BLOCK_NIDS(tmp) - 1)) {
 626                             ID id2 = ID_BLOCK_ID(tmp, ID_BLOCK_NIDS(tmp) - 1);
 627                             Datum k3;
 628
 629                             ldbm_datum_init( k3 );
 630
 631                             --ID_BLOCK_NIDS(tmp);
 632                             /* This must succeed since we just popped one
 633                              * ID off the end of it.
 634                              */
 635                             rc = idl_insert( &tmp, id, db->dbc_maxids );
 636
 637                                 k3.dptr = ch_malloc(k2.dsize);
 638                                 k3.dsize = k2.dsize;
 639                                 AC_MEMCPY(k3.dptr, k2.dptr, k3.dsize);
 640                             if ( (rc = idl_store( be, db, k3, tmp )) != 0 ) {
 641 #ifdef NEW_LOGGING
 642                                 LDAP_LOG(( "cache", LDAP_LEVEL_ERR,
 643                                                "idl_insert_key: idl_store returned %d\n", rc ));
 644 #else
 645                                 Debug( LDAP_DEBUG_ANY,
 646                             "idl_insert_key: idl_store returned %d\n", rc, 0, 0 );
 647 #endif
 648
 649                             }
 650
 651                                 free( k3.dptr );
 652
 653                             id = id2;
 654                             /* This new id will necessarily be inserted
 655                              * as the first id of the next block by the
 656                              * following switch() statement.
 657                              */
 658                         }
 659
 660                         switch ( (rc = idl_insert( &tmp2, id,
 661                             db->dbc_maxids )) ) {
 662                         case 1:         /* id inserted first in block */
 663                                 rc = idl_change_first( be, db, key, idl,
 664                                     i + 1, k2, tmp2 );
 665                                 /* FALL */
 666
 667                         case 2:         /* id already there - how? */
 668                         case 0:         /* id inserted: this can never be
 669                                          * the result of idl_insert, because
 670                                          * we guaranteed that idl_change_first
 671                                          * will always be called.
 672                                          */
 673                                 if ( rc == 2 ) {
 674 #ifdef NEW_LOGGING
 675                                         LDAP_LOG(( "cache", LDAP_LEVEL_INFO,
 676                                                    "idl_insert_key: id %ld is already in next block\n",
 677                                                    id ));
 678 #else
 679                                         Debug( LDAP_DEBUG_ANY,
 680                                             "idl_insert_key: id %ld already in next block\n",
 681                                             id, 0, 0 );
 682 #endif
 683
 684                                 }
 685
 686                                 idl_free( tmp );
 687                                 idl_free( tmp2 );
 688                                 idl_free( idl );
 689                                 return( 0 );
 690
 691                         case 3:         /* split the original block */
 692                                 break;
 693                         }
 694
 695                         idl_free( tmp2 );
 696                 }
 697
 698 split:
 699                 /*
 700                  * must split the block, write both new blocks + update
 701                  * and write the indirect header block.
 702                  */
 703
 704                 rc = 0; /* optimistic */
 705
 706
 707                 /* count how many indirect blocks *//* XXX linear count XXX */
 708                 for ( j = 0; !ID_BLOCK_NOID(idl, j); j++ )
 709                         ;       /* NULL */
 710
 711                 /* check it against all-id thresholed */
 712                 if ( j + 1 > db->dbc_maxindirect ) {
 713                         /*
 714                          * we've passed the all-id threshold, meaning
 715                          * that this set of blocks should be replaced
 716                          * by a single "all-id" block.  our job: delete
 717                          * all the indirect blocks, and replace the header
 718                          * block by an all-id block.
 719                          */
 720
 721                         /* delete all indirect blocks */
 722                         for ( j = 0; !ID_BLOCK_NOID(idl, j); j++ ) {
 723                                 cont_id( &k2, ID_BLOCK_ID(idl, j) );
 724
 725                                 rc = ldbm_cache_delete( db, k2 );
 726                         }
 727
 728                         /* store allid block in place of header block */
 729                         idl_free( idl );
 730                         idl = idl_allids( be );
 731                         rc = idl_store( be, db, key, idl );
 732
 733                         cont_free( &k2 );
 734                         idl_free( idl );
 735                         idl_free( tmp );
 736                         return( rc );
 737                 }
 738
 739                 idl_split_block( tmp, id, &tmp2, &tmp3 );
 740                 idl_free( tmp );
 741
 742                 /* create a new updated indirect header block */
 743                 tmp = idl_alloc( ID_BLOCK_NMAX(idl) + 1 );
 744                 ID_BLOCK_NIDS(tmp) = ID_BLOCK_INDIRECT_VALUE;
 745                 /* everything up to the split block */
 746                 AC_MEMCPY(
 747                         (char *) &ID_BLOCK_ID(tmp, 0),
 748                         (char *) &ID_BLOCK_ID(idl, 0),
 749                     i * sizeof(ID) );
 750                 /* the two new blocks */
 751                 ID_BLOCK_ID(tmp, i) = ID_BLOCK_ID(tmp2, 0);
 752                 ID_BLOCK_ID(tmp, i + 1) = ID_BLOCK_ID(tmp3, 0);
 753                 /* everything after the split block */
 754                 AC_MEMCPY(
 755                         (char *) &ID_BLOCK_ID(tmp, i + 2),
 756                         (char *) &ID_BLOCK_ID(idl, i + 1),
 757                         (ID_BLOCK_NMAX(idl) - i - 1) * sizeof(ID) );
 758
 759                 /* store the header block */
 760                 rc = idl_store( be, db, key, tmp );
 761
 762                 /* store the first id block */
 763                 cont_id( &k2, ID_BLOCK_ID(tmp2, 0) );
 764                 rc = idl_store( be, db, k2, tmp2 );
 765
 766                 /* store the second id block */
 767                 cont_id( &k2, ID_BLOCK_ID(tmp3, 0) );
 768                 rc = idl_store( be, db, k2, tmp3 );
 769
 770                 idl_free( tmp2 );
 771                 idl_free( tmp3 );
 772                 break;
 773         }
 774
 775         cont_free( &k2 );
 776         idl_free( tmp );
 777         idl_free( idl );
 778         return( rc );
 779 }
 780
 781
 782 /*
 783  * idl_insert - insert an id into an id list.
 784  *
 785  *      returns
 786  *              0       id inserted
 787  *              1       id inserted, first id in block has changed
 788  *              2       id not inserted, already there
 789  *              3       id not inserted, block must be split
 790  */
 791 int
 792 idl_insert( ID_BLOCK **idl, ID id, unsigned int maxids )
 793 {
 794         unsigned int    i;
 795
 796         if ( ID_BLOCK_ALLIDS( *idl ) ) {
 797                 return( 2 );    /* already there */
 798         }
 799
 800         /* is it already there? */
 801         i = idl_find(*idl, id);
 802         if ( ID_BLOCK_ID(*idl, i) == id ) {
 803                 return( 2 );    /* already there */
 804         }
 805         if ( ID_BLOCK_ID(*idl, i) < id )
 806                 i++;
 807
 808         /* do we need to make room for it? */
 809         if ( ID_BLOCK_NIDS(*idl) == ID_BLOCK_NMAX(*idl) ) {
 810                 /* make room or indicate block needs splitting */
 811                 if ( ID_BLOCK_NMAX(*idl) >= maxids ) {
 812                         return( 3 );    /* block needs splitting */
 813                 }
 814
 815                 ID_BLOCK_NMAX(*idl) *= 2;
 816                 if ( ID_BLOCK_NMAX(*idl) > maxids ) {
 817                         ID_BLOCK_NMAX(*idl) = maxids;
 818                 }
 819                 *idl = (ID_BLOCK *) ch_realloc( (char *) *idl,
 820                     (ID_BLOCK_NMAX(*idl) + ID_BLOCK_IDS_OFFSET) * sizeof(ID) );
 821         }
 822
 823         /* make a slot for the new id */
 824         AC_MEMCPY( &ID_BLOCK_ID(*idl, i+1), &ID_BLOCK_ID(*idl, i),
 825                     (ID_BLOCK_NIDS(*idl) - i) * sizeof(ID) );
 826
 827         ID_BLOCK_ID(*idl, i) = id;
 828         ID_BLOCK_NIDS(*idl)++;
 829         (void) memset(
 830                 (char *) &ID_BLOCK_ID((*idl), ID_BLOCK_NIDS(*idl)),
 831                 '\0',
 832             (ID_BLOCK_NMAX(*idl) - ID_BLOCK_NIDS(*idl)) * sizeof(ID) );
 833
 834 #ifdef LDBM_DEBUG_IDL
 835         idl_check(*idl);
 836 #endif
 837
 838         return( i == 0 ? 1 : 0 );       /* inserted - first id changed or not */
 839 }
 840
 841
 842 int
 843 idl_delete_key (
 844         Backend         *be,
 845         DBCache  *db,
 846         Datum           key,
 847         ID              id
 848 )
 849 {
 850         Datum  data;
 851         ID_BLOCK *idl;
 852         unsigned i;
 853         int j, nids;
 854
 855         if ( (idl = idl_fetch_one( be, db, key ) ) == NULL )
 856         {
 857                 /* It wasn't found.  Hmm... */
 858                 return -1;
 859         }
 860
 861         if ( ID_BLOCK_ALLIDS( idl ) ) {
 862                 idl_free( idl );
 863                 return 0;
 864         }
 865
 866         if ( ! ID_BLOCK_INDIRECT( idl ) ) {
 867                 for ( i=0; i < ID_BLOCK_NIDS(idl); i++ ) {
 868                         if ( ID_BLOCK_ID(idl, i) == id ) {
 869                                 if( --ID_BLOCK_NIDS(idl) == 0 ) {
 870                                         ldbm_cache_delete( db, key );
 871
 872                                 } else {
 873                                         AC_MEMCPY(
 874                                                 &ID_BLOCK_ID(idl, i),
 875                                                 &ID_BLOCK_ID(idl, i+1),
 876                                                 (ID_BLOCK_NIDS(idl)-i) * sizeof(ID) );
 877
 878                                         ID_BLOCK_ID(idl, ID_BLOCK_NIDS(idl)) = NOID;
 879
 880                                         idl_store( be, db, key, idl );
 881                                 }
 882
 883                                 idl_free( idl );
 884                                 return 0;
 885                         }
 886                         /*  We didn't find the ID.  Hmmm... */
 887                 }
 888                 idl_free( idl );
 889                 return -1;
 890         }
 891
 892         /* We have to go through an indirect block and find the ID
 893            in the list of IDL's
 894            */
 895         for ( nids = 0; !ID_BLOCK_NOID(idl, nids); nids++ )
 896                 ;       /* NULL */
 897
 898         cont_alloc( &data, &key );
 899
 900         for ( j = 0; !ID_BLOCK_NOID(idl, j); j++ )
 901         {
 902                 ID_BLOCK *tmp;
 903                 cont_id( &data, ID_BLOCK_ID(idl, j) );
 904
 905                 if ( (tmp = idl_fetch_one( be, db, data )) == NULL ) {
 906 #ifdef NEW_LOGGING
 907                         LDAP_LOG(( "cache", LDAP_LEVEL_INFO,
 908                                    "idl_delete_key: idl_fetch_one returned NULL\n" ));
 909 #else
 910                         Debug( LDAP_DEBUG_ANY,
 911                             "idl_delete_key: idl_fetch of returned NULL\n", 0, 0, 0 );
 912 #endif
 913
 914                         continue;
 915                 }
 916                 /*
 917                    Now try to find the ID in tmp
 918                 */
 919                 for ( i=0; i < ID_BLOCK_NIDS(tmp); i++ )
 920                 {
 921                         if ( ID_BLOCK_ID(tmp, i) == id )
 922                         {
 923                                 AC_MEMCPY(
 924                                         &ID_BLOCK_ID(tmp, i),
 925                                         &ID_BLOCK_ID(tmp, i+1),
 926                                         (ID_BLOCK_NIDS(tmp)-(i+1)) * sizeof(ID));
 927                                 ID_BLOCK_ID(tmp, ID_BLOCK_NIDS(tmp)-1 ) = NOID;
 928                                 ID_BLOCK_NIDS(tmp)--;
 929
 930                                 if ( ID_BLOCK_NIDS(tmp) ) {
 931                                         idl_store ( be, db, data, tmp );
 932
 933                                 } else {
 934                                         ldbm_cache_delete( db, data );
 935                                         AC_MEMCPY(
 936                                                 &ID_BLOCK_ID(idl, j),
 937                                                 &ID_BLOCK_ID(idl, j+1),
 938                                                 (nids-(j+1)) * sizeof(ID));
 939                                         ID_BLOCK_ID(idl, nids-1) = NOID;
 940                                         nids--;
 941                                         if ( ! nids )
 942                                                 ldbm_cache_delete( db, key );
 943                                         else
 944                                                 idl_store( be, db, key, idl );
 945                                 }
 946                                 idl_free( tmp );
 947                                 cont_free( &data );
 948                                 idl_free( idl );
 949                                 return 0;
 950                         }
 951                 }
 952                 idl_free( tmp );
 953         }
 954
 955         cont_free( &data );
 956         idl_free( idl );
 957         return -1;
 958 }
 959
 960
 961 /* return a duplicate of a single ID_BLOCK */
 962 static ID_BLOCK *
 963 idl_dup( ID_BLOCK *idl )
 964 {
 965         ID_BLOCK        *new;
 966
 967         if ( idl == NULL ) {
 968                 return( NULL );
 969         }
 970
 971         new = idl_alloc( ID_BLOCK_NMAX(idl) );
 972
 973         AC_MEMCPY(
 974                 (char *) new,
 975                 (char *) idl,
 976                 (ID_BLOCK_NMAX(idl) + ID_BLOCK_IDS_OFFSET) * sizeof(ID) );
 977
 978 #ifdef LDBM_DEBUG_IDL
 979         idl_check(new);
 980 #endif
 981
 982         return( new );
 983 }
 984
 985
 986 /* return the smaller ID_BLOCK */
 987 static ID_BLOCK *
 988 idl_min( ID_BLOCK *a, ID_BLOCK *b )
 989 {
 990         return( ID_BLOCK_NIDS(a) > ID_BLOCK_NIDS(b) ? b : a );
 991 }
 992
 993
 994 /*
 995  * idl_intersection - return a intersection b
 996  */
 997 ID_BLOCK *
 998 idl_intersection(
 999     Backend     *be,
1000     ID_BLOCK    *a,
1001     ID_BLOCK    *b
1002 )
1003 {
1004         unsigned int    ai, bi, ni;
1005         ID_BLOCK                *n;
1006
1007         if ( a == NULL || b == NULL ) {
1008                 return( NULL );
1009         }
1010         if ( ID_BLOCK_ALLIDS( a ) ) {
1011                 return( idl_dup( b ) );
1012         }
1013         if ( ID_BLOCK_ALLIDS( b ) ) {
1014                 return( idl_dup( a ) );
1015         }
1016
1017         n = idl_dup( idl_min( a, b ) );
1018
1019 #ifdef LDBM_DEBUG_IDL
1020         idl_check(a);
1021         idl_check(b);
1022 #endif
1023
1024         for ( ni = 0, ai = 0, bi = 0; ai < ID_BLOCK_NIDS(a); ai++ ) {
1025                 for ( ;
1026                         bi < ID_BLOCK_NIDS(b) && ID_BLOCK_ID(b, bi) < ID_BLOCK_ID(a, ai);
1027                         bi++ )
1028                 {
1029                         ;       /* NULL */
1030                 }
1031
1032                 if ( bi == ID_BLOCK_NIDS(b) ) {
1033                         break;
1034                 }
1035
1036                 if ( ID_BLOCK_ID(b, bi) == ID_BLOCK_ID(a, ai) ) {
1037                         ID_BLOCK_ID(n, ni++) = ID_BLOCK_ID(a, ai);
1038                 }
1039         }
1040
1041         if ( ni == 0 ) {
1042                 idl_free( n );
1043                 return( NULL );
1044         }
1045         ID_BLOCK_NIDS(n) = ni;
1046
1047 #ifdef LDBM_DEBUG_IDL
1048         idl_check(n);
1049 #endif
1050
1051         return( n );
1052 }
1053
1054
1055 /*
1056  * idl_union - return a union b
1057  */
1058 ID_BLOCK *
1059 idl_union(
1060     Backend     *be,
1061     ID_BLOCK    *a,
1062     ID_BLOCK    *b
1063 )
1064 {
1065         unsigned int    ai, bi, ni;
1066         ID_BLOCK                *n;
1067
1068         if ( a == NULL ) {
1069                 return( idl_dup( b ) );
1070         }
1071         if ( b == NULL ) {
1072                 return( idl_dup( a ) );
1073         }
1074         if ( ID_BLOCK_ALLIDS( a ) || ID_BLOCK_ALLIDS( b ) ) {
1075                 return( idl_allids( be ) );
1076         }
1077
1078 #ifdef LDBM_DEBUG_IDL
1079         idl_check(a);
1080         idl_check(b);
1081 #endif
1082
1083         if ( ID_BLOCK_NIDS(b) < ID_BLOCK_NIDS(a) ) {
1084                 n = a;
1085                 a = b;
1086                 b = n;
1087         }
1088
1089         n = idl_alloc( ID_BLOCK_NIDS(a) + ID_BLOCK_NIDS(b) );
1090
1091         for ( ni = 0, ai = 0, bi = 0;
1092                 ai < ID_BLOCK_NIDS(a) && bi < ID_BLOCK_NIDS(b);
1093                 )
1094         {
1095                 if ( ID_BLOCK_ID(a, ai) < ID_BLOCK_ID(b, bi) ) {
1096                         ID_BLOCK_ID(n, ni++) = ID_BLOCK_ID(a, ai++);
1097
1098                 } else if ( ID_BLOCK_ID(b, bi) < ID_BLOCK_ID(a, ai) ) {
1099                         ID_BLOCK_ID(n, ni++) = ID_BLOCK_ID(b, bi++);
1100
1101                 } else {
1102                         ID_BLOCK_ID(n, ni++) = ID_BLOCK_ID(a, ai);
1103                         ai++, bi++;
1104                 }
1105         }
1106
1107         for ( ; ai < ID_BLOCK_NIDS(a); ai++ ) {
1108                 ID_BLOCK_ID(n, ni++) = ID_BLOCK_ID(a, ai);
1109         }
1110         for ( ; bi < ID_BLOCK_NIDS(b); bi++ ) {
1111                 ID_BLOCK_ID(n, ni++) = ID_BLOCK_ID(b, bi);
1112         }
1113         ID_BLOCK_NIDS(n) = ni;
1114
1115 #ifdef LDBM_DEBUG_IDL
1116         idl_check(n);
1117 #endif
1118
1119         return( n );
1120 }
1121
1122
1123 /*
1124  * idl_notin - return a intersection ~b (or a minus b)
1125  */
1126 ID_BLOCK *
1127 idl_notin(
1128     Backend     *be,
1129     ID_BLOCK    *a,
1130     ID_BLOCK    *b
1131 )
1132 {
1133         unsigned int    ni, ai, bi;
1134         ID_BLOCK                *n;
1135
1136         if ( a == NULL ) {
1137                 return( NULL );
1138         }
1139         if ( b == NULL || ID_BLOCK_ALLIDS( b )) {
1140                 return( idl_dup( a ) );
1141         }
1142
1143         if ( ID_BLOCK_ALLIDS( a ) ) {
1144                 n = idl_alloc( SLAPD_LDBM_MIN_MAXIDS );
1145                 ni = 0;
1146
1147                 for ( ai = 1, bi = 0;
1148                         ai < ID_BLOCK_NIDS(a) && ni < ID_BLOCK_NMAX(n) && bi < ID_BLOCK_NMAX(b);
1149                         ai++ )
1150                 {
1151                         if ( ID_BLOCK_ID(b, bi) == ai ) {
1152                                 bi++;
1153                         } else {
1154                                 ID_BLOCK_ID(n, ni++) = ai;
1155                         }
1156                 }
1157
1158                 for ( ; ai < ID_BLOCK_NIDS(a) && ni < ID_BLOCK_NMAX(n); ai++ ) {
1159                         ID_BLOCK_ID(n, ni++) = ai;
1160                 }
1161
1162                 if ( ni == ID_BLOCK_NMAX(n) ) {
1163                         idl_free( n );
1164                         return( idl_allids( be ) );
1165                 } else {
1166                         ID_BLOCK_NIDS(n) = ni;
1167                         return( n );
1168                 }
1169         }
1170
1171         n = idl_dup( a );
1172
1173         ni = 0;
1174         for ( ai = 0, bi = 0; ai < ID_BLOCK_NIDS(a); ai++ ) {
1175                 for ( ;
1176                         bi < ID_BLOCK_NIDS(b) && ID_BLOCK_ID(b, bi) < ID_BLOCK_ID(a, ai);
1177                     bi++ )
1178                 {
1179                         ;       /* NULL */
1180                 }
1181
1182                 if ( bi == ID_BLOCK_NIDS(b) ) {
1183                         break;
1184                 }
1185
1186                 if ( ID_BLOCK_ID(b, bi) != ID_BLOCK_ID(a, ai) ) {
1187                         ID_BLOCK_ID(n, ni++) = ID_BLOCK_ID(a, ai);
1188                 }
1189         }
1190
1191         for ( ; ai < ID_BLOCK_NIDS(a); ai++ ) {
1192                 ID_BLOCK_ID(n, ni++) = ID_BLOCK_ID(a, ai);
1193         }
1194         ID_BLOCK_NIDS(n) = ni;
1195
1196 #ifdef LDBM_DEBUG_IDL
1197         idl_check(n);
1198 #endif
1199
1200         return( n );
1201 }
1202
1203 /*      return the first ID in the block
1204  *      if ALLIDS block
1205  *              NIDS > 1 return 1
1206  *              otherwise return NOID
1207  *      otherwise return first ID
1208  *
1209  *      cursor is set to 1
1210  */
1211 ID
1212 idl_firstid( ID_BLOCK *idl, ID *cursor )
1213 {
1214         *cursor = 1;
1215
1216         if ( idl == NULL || ID_BLOCK_NIDS(idl) == 0 ) {
1217                 return( NOID );
1218         }
1219
1220         if ( ID_BLOCK_ALLIDS( idl ) ) {
1221                 return( ID_BLOCK_NIDS(idl) > 1 ? 1 : NOID );
1222         }
1223
1224         return( ID_BLOCK_ID(idl, 0) );
1225 }
1226
1227 /*      return next ID
1228  *      if ALLIDS block, cursor is id.
1229  *              increment id
1230  *              if id < NIDS return id
1231  *              otherwise NOID.
1232  *      otherwise cursor is index into block
1233  *              if index < nids
1234  *                      return id at index then increment
1235  */
1236 ID
1237 idl_nextid( ID_BLOCK *idl, ID *cursor )
1238 {
1239         if ( ID_BLOCK_ALLIDS( idl ) ) {
1240                 if( ++(*cursor) < ID_BLOCK_NIDS(idl) ) {
1241                         return *cursor;
1242                 } else {
1243                         return NOID;
1244                 }
1245         }
1246
1247         if ( *cursor < ID_BLOCK_NIDS(idl) ) {
1248                 return( ID_BLOCK_ID(idl, (*cursor)++) );
1249         }
1250
1251         return( NOID );
1252 }