git.sur5r.net Git - openldap/blob - servers/slapd/back-ldbm/idl.c

   1 /* idl.c - ldap id list handling routines */
   2 /* $OpenLDAP$ */
   3 /*
   4  * Copyright 1998-2000 The OpenLDAP Foundation, All Rights Reserved.
   5  * COPYING RESTRICTIONS APPLY, see COPYRIGHT file
   6  */
   7
   8 #include "portable.h"
   9
  10 #include <stdio.h>
  11
  12 #include <ac/string.h>
  13 #include <ac/socket.h>
  14
  15 #include "slap.h"
  16 #include "back-ldbm.h"
  17
  18 static ID_BLOCK* idl_dup( ID_BLOCK *idl );
  19
  20 static void cont_alloc( Datum *cont, Datum *key )
  21 {
  22         ldbm_datum_init( *cont );
  23         cont->dsize = 1 + sizeof(ID) + key->dsize;
  24         cont->dptr = ch_malloc( cont->dsize );
  25
  26         * (unsigned char *) cont->dptr = SLAP_INDEX_CONT_PREFIX;
  27
  28         AC_MEMCPY( &((unsigned char *)cont->dptr)[1 + sizeof(ID)],
  29                 key->dptr, key->dsize );
  30 }
  31
  32 static void cont_id( Datum *cont, ID id )
  33 {
  34         int i;
  35
  36         for( i=1; i <= sizeof(id); i++) {
  37                 ((unsigned char *)cont->dptr)[i] = (unsigned char)(id & 0xFF);
  38                 id >>= 8;
  39         }
  40
  41 }
  42
  43 static void cont_free( Datum *cont )
  44 {
  45         ch_free( cont->dptr );
  46 }
  47
  48 #ifdef LDBM_DEBUG_IDL
  49 static void idl_check(ID_BLOCK *idl)
  50 {
  51         int i;
  52         ID_BLOCK last;
  53
  54         if( ID_BLOCK_INDIRECT(idl) || ID_BLOCK_ALLIDS(idl)
  55                 || ID_BLOCK_NIDS(idl) <= 1 )
  56         {
  57                 return;
  58         }
  59
  60         for( last = ID_BLOCK_ID(idl, 0), i = 1;
  61                 i < ID_BLOCK_NIDS(idl);
  62                 last = ID_BLOCK_ID(idl, i), i++ )
  63         {
  64                 assert (last < ID_BLOCK_ID(idl, i) );
  65         }
  66 }
  67 #endif
  68
  69 /* Allocate an ID_BLOCK with room for nids ids */
  70 ID_BLOCK *
  71 idl_alloc( unsigned int nids )
  72 {
  73         ID_BLOCK        *new;
  74
  75         /* nmax + nids + space for the ids */
  76         new = (ID_BLOCK *) ch_calloc( (ID_BLOCK_IDS_OFFSET + nids), sizeof(ID) );
  77         ID_BLOCK_NMAX(new) = nids;
  78         ID_BLOCK_NIDS(new) = 0;
  79
  80         return( new );
  81 }
  82
  83
  84 /* Allocate an empty ALLIDS ID_BLOCK */
  85 ID_BLOCK        *
  86 idl_allids( Backend *be )
  87 {
  88         ID_BLOCK        *idl;
  89         ID              id;
  90
  91         idl = idl_alloc( 0 );
  92         ID_BLOCK_NMAX(idl) = ID_BLOCK_ALLIDS_VALUE;
  93         if ( next_id_get( be, &id ) ) {
  94                 return NULL;
  95         }
  96         ID_BLOCK_NIDS(idl) = id;
  97
  98         return( idl );
  99 }
 100
 101 /* Free an ID_BLOCK */
 102 void
 103 idl_free( ID_BLOCK *idl )
 104 {
 105         if ( idl == NULL ) {
 106 #ifdef NEW_LOGGING
 107                 LDAP_LOG(( "cache", LDAP_LEVEL_INFO,
 108                            "idl_free: called with NULL pointer\n" ));
 109 #else
 110                 Debug( LDAP_DEBUG_TRACE,
 111                         "idl_free: called with NULL pointer\n",
 112                         0, 0, 0 );
 113 #endif
 114
 115                 return;
 116         }
 117
 118         free( (char *) idl );
 119 }
 120
 121
 122 /* Fetch an single ID_BLOCK from the cache */
 123 static ID_BLOCK *
 124 idl_fetch_one(
 125     Backend             *be,
 126     DBCache     *db,
 127     Datum               key
 128 )
 129 {
 130         Datum   data;
 131         ID_BLOCK        *idl;
 132
 133         /* Debug( LDAP_DEBUG_TRACE, "=> idl_fetch_one\n", 0, 0, 0 ); */
 134
 135         data = ldbm_cache_fetch( db, key );
 136
 137         if( data.dptr == NULL ) {
 138                 return NULL;
 139         }
 140
 141         idl = idl_dup((ID_BLOCK *) data.dptr);
 142
 143         ldbm_datum_free( db->dbc_db, data );
 144
 145         return idl;
 146 }
 147
 148
 149 /* Fetch a set of ID_BLOCKs from the cache
 150  *      if not INDIRECT
 151  *              if block return is an ALLIDS block,
 152  *                      return an new ALLIDS block
 153  *              otherwise
 154  *                      return block
 155  *      construct super block from all blocks referenced by INDIRECT block
 156  *      return super block
 157  */
 158 ID_BLOCK *
 159 idl_fetch(
 160     Backend             *be,
 161     DBCache     *db,
 162     Datum               key
 163 )
 164 {
 165         Datum   data;
 166         ID_BLOCK        *idl;
 167         ID_BLOCK        **tmp;
 168         int     i, nids;
 169
 170         idl = idl_fetch_one( be, db, key );
 171
 172         if ( idl == NULL ) {
 173                 return NULL;
 174         }
 175
 176         if ( ID_BLOCK_ALLIDS(idl) ) {
 177                 /* all ids block */
 178                 /* make sure we have the current value of highest id */
 179                 idl_free( idl );
 180                 idl = idl_allids( be );
 181
 182                 return( idl );
 183         }
 184
 185         if ( ! ID_BLOCK_INDIRECT( idl ) ) {
 186                 /* regular block */
 187                 return( idl );
 188         }
 189
 190         /*
 191          * this is an indirect block which points to other blocks.
 192          * we need to read in all the blocks it points to and construct
 193          * a big id list containing all the ids, which we will return.
 194          */
 195
 196         /* count the number of blocks & allocate space for pointers to them */
 197         for ( i = 0; !ID_BLOCK_NOID(idl, i); i++ )
 198                 ;       /* NULL */
 199         tmp = (ID_BLOCK **) ch_malloc( (i + 1) * sizeof(ID_BLOCK *) );
 200
 201         /* read in all the blocks */
 202         cont_alloc( &data, &key );
 203         nids = 0;
 204         for ( i = 0; !ID_BLOCK_NOID(idl, i); i++ ) {
 205                 cont_id( &data, ID_BLOCK_ID(idl, i) );
 206
 207                 if ( (tmp[i] = idl_fetch_one( be, db, data )) == NULL ) {
 208 #ifdef NEW_LOGGING
 209                         LDAP_LOG(( "cache", LDAP_LEVEL_INFO,
 210                                    "idl_fetch: idl_fetch_one returned NULL\n" ));
 211 #else
 212                         Debug( LDAP_DEBUG_ANY,
 213                             "idl_fetch: one returned NULL\n", 0, 0, 0 );
 214 #endif
 215
 216                         continue;
 217                 }
 218
 219                 nids += ID_BLOCK_NIDS(tmp[i]);
 220         }
 221         tmp[i] = NULL;
 222         cont_free( &data );
 223         idl_free( idl );
 224
 225         /* allocate space for the big block */
 226         idl = idl_alloc( nids );
 227         ID_BLOCK_NIDS(idl) = nids;
 228         nids = 0;
 229
 230         /* copy in all the ids from the component blocks */
 231         for ( i = 0; tmp[i] != NULL; i++ ) {
 232                 if ( tmp[i] == NULL ) {
 233                         continue;
 234                 }
 235
 236                 AC_MEMCPY(
 237                         (char *) &ID_BLOCK_ID(idl, nids),
 238                         (char *) &ID_BLOCK_ID(tmp[i], 0),
 239                         ID_BLOCK_NIDS(tmp[i]) * sizeof(ID) );
 240                 nids += ID_BLOCK_NIDS(tmp[i]);
 241
 242                 idl_free( tmp[i] );
 243         }
 244         free( (char *) tmp );
 245
 246 #ifdef LDBM_DEBUG_IDL
 247         idl_check(idl);
 248 #endif
 249
 250 #ifdef NEW_LOGGING
 251         LDAP_LOG(( "cache", LDAP_LEVEL_ENTRY,
 252                    "idl_fetch: %ld ids (%ld max)\n",
 253                    ID_BLOCK_NIDS(idl), ID_BLOCK_NMAX(idl) ));
 254 #else
 255         Debug( LDAP_DEBUG_TRACE, "<= idl_fetch %ld ids (%ld max)\n",
 256                ID_BLOCK_NIDS(idl), ID_BLOCK_NMAX(idl), 0 );
 257 #endif
 258
 259         return( idl );
 260 }
 261
 262
 263 /* store a single block */
 264 static int
 265 idl_store(
 266     Backend             *be,
 267     DBCache     *db,
 268     Datum               key,
 269     ID_BLOCK            *idl
 270 )
 271 {
 272         int     rc, flags;
 273         Datum   data;
 274         struct ldbminfo *li = (struct ldbminfo *) be->be_private;
 275
 276 #ifdef LDBM_DEBUG_IDL
 277         idl_check(idl);
 278 #endif
 279
 280         ldbm_datum_init( data );
 281
 282         /* Debug( LDAP_DEBUG_TRACE, "=> idl_store\n", 0, 0, 0 ); */
 283
 284         data.dptr = (char *) idl;
 285         data.dsize = (ID_BLOCK_IDS_OFFSET + ID_BLOCK_NMAX(idl)) * sizeof(ID);
 286
 287 #ifdef LDBM_DEBUG
 288         Statslog( LDAP_DEBUG_STATS, "<= idl_store(): rc=%d\n",
 289                 rc, 0, 0, 0, 0 );
 290 #endif
 291
 292         flags = LDBM_REPLACE;
 293         rc = ldbm_cache_store( db, key, data, flags );
 294
 295         /* Debug( LDAP_DEBUG_TRACE, "<= idl_store %d\n", rc, 0, 0 ); */
 296         return( rc );
 297 }
 298
 299 /* split the block at id
 300  *      locate ID greater than or equal to id.
 301  */
 302 static void
 303 idl_split_block(
 304     ID_BLOCK    *b,
 305     ID          id,
 306     ID_BLOCK    **right,
 307     ID_BLOCK    **left
 308 )
 309 {
 310         unsigned int    nr, nl;
 311
 312         /* find where to split the block *//* XXX linear search XXX */
 313         for ( nr = 0; nr < ID_BLOCK_NIDS(b) && id > ID_BLOCK_ID(b, nr); nr++ )
 314                 ;       /* NULL */
 315
 316         nl = ID_BLOCK_NIDS(b) - nr;
 317
 318         *right = idl_alloc( nr == 0 ? 1 : nr );
 319         *left = idl_alloc( nl + (nr == 0 ? 0 : 1));
 320
 321         /*
 322          * everything before the id being inserted in the first block
 323          * unless there is nothing, in which case the id being inserted
 324          * goes there.
 325          */
 326         if ( nr == 0 ) {
 327                 ID_BLOCK_NIDS(*right) = 1;
 328                 ID_BLOCK_ID(*right, 0) = id;
 329         } else {
 330                 AC_MEMCPY(
 331                         (char *) &ID_BLOCK_ID(*right, 0),
 332                         (char *) &ID_BLOCK_ID(b, 0),
 333                         nr * sizeof(ID) );
 334                 ID_BLOCK_NIDS(*right) = nr;
 335                 ID_BLOCK_ID(*left, 0) = id;
 336         }
 337
 338         /* the id being inserted & everything after in the second block */
 339         AC_MEMCPY(
 340                 (char *) &ID_BLOCK_ID(*left, (nr == 0 ? 0 : 1)),
 341             (char *) &ID_BLOCK_ID(b, nr),
 342                 nl * sizeof(ID) );
 343         ID_BLOCK_NIDS(*left) = nl + (nr == 0 ? 0 : 1);
 344
 345 #ifdef LDBM_DEBUG_IDL
 346         idl_check(*right);
 347         idl_check(*left);
 348 #endif
 349 }
 350
 351
 352 /*
 353  * idl_change_first - called when an indirect block's first key has
 354  * changed, meaning it needs to be stored under a new key, and the
 355  * header block pointing to it needs updating.
 356  */
 357 static int
 358 idl_change_first(
 359     Backend             *be,
 360     DBCache     *db,
 361     Datum               hkey,           /* header block key     */
 362     ID_BLOCK            *h,             /* header block         */
 363     int                 pos,            /* pos in h to update   */
 364     Datum               bkey,           /* data block key       */
 365     ID_BLOCK            *b              /* data block           */
 366 )
 367 {
 368         int     rc;
 369
 370         /* Debug( LDAP_DEBUG_TRACE, "=> idl_change_first\n", 0, 0, 0 ); */
 371
 372         /* delete old key block */
 373         if ( (rc = ldbm_cache_delete( db, bkey )) != 0 ) {
 374 #ifdef NEW_LOGGING
 375                 LDAP_LOG(( "cache", LDAP_LEVEL_INFO,
 376                            "idl_change_first: ldbm_cache_delete returned %d\n", rc ));
 377 #else
 378                 Debug( LDAP_DEBUG_ANY,
 379                     "idl_change_first: ldbm_cache_delete returned %d\n",
 380                         rc, 0, 0 );
 381 #endif
 382
 383                 return( rc );
 384         }
 385
 386         /* write block with new key */
 387         cont_id( &bkey, ID_BLOCK_ID(b, 0) );
 388
 389         if ( (rc = idl_store( be, db, bkey, b )) != 0 ) {
 390 #ifdef NEW_LOGGING
 391                 LDAP_LOG(( "cache", LDAP_LEVEL_INFO,
 392                            "idl_change_first: idl_store returned %d\n", rc ));
 393 #else
 394                 Debug( LDAP_DEBUG_ANY,
 395                     "idl_change_first: idl_store returned %d\n", rc, 0, 0 );
 396 #endif
 397
 398                 return( rc );
 399         }
 400
 401         /* update + write indirect header block */
 402         ID_BLOCK_ID(h, pos) = ID_BLOCK_ID(b, 0);
 403         if ( (rc = idl_store( be, db, hkey, h )) != 0 ) {
 404 #ifdef NEW_LOGGING
 405                 LDAP_LOG(( "cache", LDAP_LEVEL_INFO,
 406                            "idl_change_first: idl_store returned %s\n", rc ));
 407 #else
 408                 Debug( LDAP_DEBUG_ANY,
 409                     "idl_change_first: idl_store returned %d\n", rc, 0, 0 );
 410 #endif
 411
 412                 return( rc );
 413         }
 414
 415         return( 0 );
 416 }
 417
 418
 419 int
 420 idl_insert_key(
 421     Backend             *be,
 422     DBCache     *db,
 423     Datum               key,
 424     ID                  id
 425 )
 426 {
 427         int     i, j, first, rc;
 428         ID_BLOCK        *idl, *tmp, *tmp2, *tmp3;
 429         Datum   k2;
 430
 431         if ( (idl = idl_fetch_one( be, db, key )) == NULL ) {
 432                 idl = idl_alloc( 1 );
 433                 ID_BLOCK_ID(idl, ID_BLOCK_NIDS(idl)++) = id;
 434                 rc = idl_store( be, db, key, idl );
 435
 436                 idl_free( idl );
 437                 return( rc );
 438         }
 439
 440         if ( ID_BLOCK_ALLIDS( idl ) ) {
 441                 /* ALLIDS */
 442                 idl_free( idl );
 443                 return 0;
 444         }
 445
 446         if ( ! ID_BLOCK_INDIRECT( idl ) ) {
 447                 /* regular block */
 448                 switch ( idl_insert( &idl, id, db->dbc_maxids ) ) {
 449                 case 0:         /* id inserted - store the updated block */
 450                 case 1:
 451                         rc = idl_store( be, db, key, idl );
 452                         break;
 453
 454                 case 2:         /* id already there - nothing to do */
 455                         rc = 0;
 456                         break;
 457
 458                 case 3:         /* id not inserted - block must be split */
 459                         /* check threshold for marking this an all-id block */
 460                         if ( db->dbc_maxindirect < 2 ) {
 461                                 idl_free( idl );
 462                                 idl = idl_allids( be );
 463                                 rc = idl_store( be, db, key, idl );
 464                                 break;
 465                         }
 466
 467                         idl_split_block( idl, id, &tmp, &tmp2 );
 468                         idl_free( idl );
 469
 470                         /* create the header indirect block */
 471                         idl = idl_alloc( 3 );
 472                         ID_BLOCK_NMAX(idl) = 3;
 473                         ID_BLOCK_NIDS(idl) = ID_BLOCK_INDIRECT_VALUE;
 474                         ID_BLOCK_ID(idl, 0) = ID_BLOCK_ID(tmp, 0);
 475                         ID_BLOCK_ID(idl, 1) = ID_BLOCK_ID(tmp2, 0);
 476                         ID_BLOCK_ID(idl, 2) = NOID;
 477
 478                         /* store it */
 479                         rc = idl_store( be, db, key, idl );
 480
 481                         cont_alloc( &k2, &key );
 482                         cont_id( &k2, ID_BLOCK_ID(tmp, 0) );
 483
 484                         rc = idl_store( be, db, k2, tmp );
 485
 486                         cont_id( &k2, ID_BLOCK_ID(tmp2, 0) );
 487                         rc = idl_store( be, db, k2, tmp2 );
 488
 489                         cont_free( &k2 );
 490
 491                         idl_free( tmp );
 492                         idl_free( tmp2 );
 493                         break;
 494                 }
 495
 496                 idl_free( idl );
 497                 return( rc );
 498         }
 499
 500         /*
 501          * this is an indirect block which points to other blocks.
 502          * we need to read in the block into which the id should be
 503          * inserted, then insert the id and store the block.  we might
 504          * have to split the block if it is full, which means we also
 505          * need to write a new "header" block.
 506          */
 507
 508         /* select the block to try inserting into *//* XXX linear search XXX */
 509         for ( i = 0; !ID_BLOCK_NOID(idl, i) && id > ID_BLOCK_ID(idl, i); i++ )
 510                 ;       /* NULL */
 511
 512         if ( i != 0 ) {
 513                 i--;
 514                 first = 0;
 515         } else {
 516                 first = 1;
 517         }
 518
 519         /* get the block */
 520         cont_alloc( &k2, &key );
 521         cont_id( &k2, ID_BLOCK_ID(idl, i) );
 522
 523         if ( (tmp = idl_fetch_one( be, db, k2 )) == NULL ) {
 524 #ifdef NEW_LOGGING
 525                 LDAP_LOG(( "cache", LDAP_LEVEL_ERR,
 526                            "idl_insert_key: nonexistent continuation block\n" ));
 527 #else
 528                 Debug( LDAP_DEBUG_ANY, "idl_insert_key: nonexistent continuation block\n",
 529                     0, 0, 0 );
 530 #endif
 531
 532                 cont_free( &k2 );
 533                 idl_free( idl );
 534                 return( -1 );
 535         }
 536
 537         /* insert the id */
 538         switch ( idl_insert( &tmp, id, db->dbc_maxids ) ) {
 539         case 0:         /* id inserted ok */
 540                 if ( (rc = idl_store( be, db, k2, tmp )) != 0 ) {
 541 #ifdef NEW_LOGGING
 542                         LDAP_LOG(( "cache", LDAP_LEVEL_ERR,
 543                                    "ids_insert_key: idl_store returned %d\n", rc ));
 544 #else
 545                         Debug( LDAP_DEBUG_ANY,
 546                             "idl_insert_key: idl_store returned %d\n", rc, 0, 0 );
 547 #endif
 548
 549                 }
 550                 break;
 551
 552         case 1:         /* id inserted - first id in block has changed */
 553                 /*
 554                  * key for this block has changed, so we have to
 555                  * write the block under the new key, delete the
 556                  * old key block + update and write the indirect
 557                  * header block.
 558                  */
 559
 560                 rc = idl_change_first( be, db, key, idl, i, k2, tmp );
 561                 break;
 562
 563         case 2:         /* id not inserted - already there, do nothing */
 564                 rc = 0;
 565                 break;
 566
 567         case 3:         /* id not inserted - block is full */
 568                 /*
 569                  * first, see if it will fit in the next block,
 570                  * without splitting, unless we're trying to insert
 571                  * into the beginning of the first block.
 572                  */
 573
 574                 /* is there a next block? */
 575                 if ( !first && !ID_BLOCK_NOID(idl, i + 1) ) {
 576                         /* read it in */
 577                         cont_alloc( &k2, &key );
 578                         cont_id( &k2, ID_BLOCK_ID(idl, i) );
 579                         if ( (tmp2 = idl_fetch_one( be, db, k2 )) == NULL ) {
 580 #ifdef NEW_LOGGING
 581                                 LDAP_LOG(( "cache", LDAP_LEVEL_ERR,
 582                                            "idl_insert_key: idl_fetch_one returned NULL\n"));
 583 #else
 584                                 Debug( LDAP_DEBUG_ANY,
 585                                     "idl_insert_key: idl_fetch_one returned NULL\n",
 586                                     0, 0, 0 );
 587 #endif
 588
 589                                 /* split the original block */
 590                                 cont_free( &k2 );
 591                                 goto split;
 592                         }
 593
 594                         /* If the new id is less than the last id in the
 595                          * current block, it must not be put into the next
 596                          * block. Push the last id of the current block
 597                          * into the next block instead.
 598                          */
 599                         if (id < ID_BLOCK_ID(tmp, ID_BLOCK_NIDS(tmp) - 1)) {
 600                             ID id2 = ID_BLOCK_ID(tmp, ID_BLOCK_NIDS(tmp) - 1);
 601                             Datum k3;
 602
 603                             ldbm_datum_init( k3 );
 604
 605                             --ID_BLOCK_NIDS(tmp);
 606                             /* This must succeed since we just popped one
 607                              * ID off the end of it.
 608                              */
 609                             rc = idl_insert( &tmp, id, db->dbc_maxids );
 610
 611                                 k3.dptr = ch_malloc(k2.dsize);
 612                                 k3.dsize = k2.dsize;
 613                                 AC_MEMCPY(k3.dptr, k2.dptr, k3.dsize);
 614                             if ( (rc = idl_store( be, db, k3, tmp )) != 0 ) {
 615 #ifdef NEW_LOGGING
 616                                 LDAP_LOG(( "cache", LDAP_LEVEL_ERR,
 617                                                "idl_insert_key: idl_store returned %d\n", rc ));
 618 #else
 619                                 Debug( LDAP_DEBUG_ANY,
 620                             "idl_insert_key: idl_store returned %d\n", rc, 0, 0 );
 621 #endif
 622
 623                             }
 624
 625                                 free( k3.dptr );
 626
 627                             id = id2;
 628                             /* This new id will necessarily be inserted
 629                              * as the first id of the next block by the
 630                              * following switch() statement.
 631                              */
 632                         }
 633
 634                         switch ( (rc = idl_insert( &tmp2, id,
 635                             db->dbc_maxids )) ) {
 636                         case 1:         /* id inserted first in block */
 637                                 rc = idl_change_first( be, db, key, idl,
 638                                     i + 1, k2, tmp2 );
 639                                 /* FALL */
 640
 641                         case 2:         /* id already there - how? */
 642                         case 0:         /* id inserted: this can never be
 643                                          * the result of idl_insert, because
 644                                          * we guaranteed that idl_change_first
 645                                          * will always be called.
 646                                          */
 647                                 if ( rc == 2 ) {
 648 #ifdef NEW_LOGGING
 649                                         LDAP_LOG(( "cache", LDAP_LEVEL_INFO,
 650                                                    "idl_insert_key: id %ld is already in next block\n",
 651                                                    id ));
 652 #else
 653                                         Debug( LDAP_DEBUG_ANY,
 654                                             "idl_insert_key: id %ld already in next block\n",
 655                                             id, 0, 0 );
 656 #endif
 657
 658                                 }
 659
 660                                 idl_free( tmp );
 661                                 idl_free( tmp2 );
 662                                 idl_free( idl );
 663                                 return( 0 );
 664
 665                         case 3:         /* split the original block */
 666                                 break;
 667                         }
 668
 669                         idl_free( tmp2 );
 670                 }
 671
 672 split:
 673                 /*
 674                  * must split the block, write both new blocks + update
 675                  * and write the indirect header block.
 676                  */
 677
 678                 rc = 0; /* optimistic */
 679
 680
 681                 /* count how many indirect blocks *//* XXX linear count XXX */
 682                 for ( j = 0; !ID_BLOCK_NOID(idl, j); j++ )
 683                         ;       /* NULL */
 684
 685                 /* check it against all-id thresholed */
 686                 if ( j + 1 > db->dbc_maxindirect ) {
 687                         /*
 688                          * we've passed the all-id threshold, meaning
 689                          * that this set of blocks should be replaced
 690                          * by a single "all-id" block.  our job: delete
 691                          * all the indirect blocks, and replace the header
 692                          * block by an all-id block.
 693                          */
 694
 695                         /* delete all indirect blocks */
 696                         for ( j = 0; !ID_BLOCK_NOID(idl, j); j++ ) {
 697                                 cont_id( &k2, ID_BLOCK_ID(idl, j) );
 698
 699                                 rc = ldbm_cache_delete( db, k2 );
 700                         }
 701
 702                         /* store allid block in place of header block */
 703                         idl_free( idl );
 704                         idl = idl_allids( be );
 705                         rc = idl_store( be, db, key, idl );
 706
 707                         cont_free( &k2 );
 708                         idl_free( idl );
 709                         idl_free( tmp );
 710                         return( rc );
 711                 }
 712
 713                 idl_split_block( tmp, id, &tmp2, &tmp3 );
 714                 idl_free( tmp );
 715
 716                 /* create a new updated indirect header block */
 717                 tmp = idl_alloc( ID_BLOCK_NMAX(idl) + 1 );
 718                 ID_BLOCK_NIDS(tmp) = ID_BLOCK_INDIRECT_VALUE;
 719                 /* everything up to the split block */
 720                 AC_MEMCPY(
 721                         (char *) &ID_BLOCK_ID(tmp, 0),
 722                         (char *) &ID_BLOCK_ID(idl, 0),
 723                     i * sizeof(ID) );
 724                 /* the two new blocks */
 725                 ID_BLOCK_ID(tmp, i) = ID_BLOCK_ID(tmp2, 0);
 726                 ID_BLOCK_ID(tmp, i + 1) = ID_BLOCK_ID(tmp3, 0);
 727                 /* everything after the split block */
 728                 AC_MEMCPY(
 729                         (char *) &ID_BLOCK_ID(tmp, i + 2),
 730                         (char *) &ID_BLOCK_ID(idl, i + 1),
 731                         (ID_BLOCK_NMAX(idl) - i - 1) * sizeof(ID) );
 732
 733                 /* store the header block */
 734                 rc = idl_store( be, db, key, tmp );
 735
 736                 /* store the first id block */
 737                 cont_id( &k2, ID_BLOCK_ID(tmp2, 0) );
 738                 rc = idl_store( be, db, k2, tmp2 );
 739
 740                 /* store the second id block */
 741                 cont_id( &k2, ID_BLOCK_ID(tmp3, 0) );
 742                 rc = idl_store( be, db, k2, tmp3 );
 743
 744                 idl_free( tmp2 );
 745                 idl_free( tmp3 );
 746                 break;
 747         }
 748
 749         cont_free( &k2 );
 750         idl_free( tmp );
 751         idl_free( idl );
 752         return( rc );
 753 }
 754
 755
 756 /*
 757  * idl_insert - insert an id into an id list.
 758  *
 759  *      returns
 760  *              0       id inserted
 761  *              1       id inserted, first id in block has changed
 762  *              2       id not inserted, already there
 763  *              3       id not inserted, block must be split
 764  */
 765 int
 766 idl_insert( ID_BLOCK **idl, ID id, unsigned int maxids )
 767 {
 768         unsigned int    i;
 769
 770         if ( ID_BLOCK_ALLIDS( *idl ) ) {
 771                 return( 2 );    /* already there */
 772         }
 773
 774         /* is it already there? *//* XXX linear search XXX */
 775         for ( i = 0; i < ID_BLOCK_NIDS(*idl) && id > ID_BLOCK_ID(*idl, i); i++ ) {
 776                 ;       /* NULL */
 777         }
 778         if ( i < ID_BLOCK_NIDS(*idl) && ID_BLOCK_ID(*idl, i) == id ) {
 779                 return( 2 );    /* already there */
 780         }
 781
 782         /* do we need to make room for it? */
 783         if ( ID_BLOCK_NIDS(*idl) == ID_BLOCK_NMAX(*idl) ) {
 784                 /* make room or indicate block needs splitting */
 785                 if ( ID_BLOCK_NMAX(*idl) >= maxids ) {
 786                         return( 3 );    /* block needs splitting */
 787                 }
 788
 789                 ID_BLOCK_NMAX(*idl) *= 2;
 790                 if ( ID_BLOCK_NMAX(*idl) > maxids ) {
 791                         ID_BLOCK_NMAX(*idl) = maxids;
 792                 }
 793                 *idl = (ID_BLOCK *) ch_realloc( (char *) *idl,
 794                     (ID_BLOCK_NMAX(*idl) + ID_BLOCK_IDS_OFFSET) * sizeof(ID) );
 795         }
 796
 797         /* make a slot for the new id */
 798         AC_MEMCPY( &ID_BLOCK_ID(*idl, i+1), &ID_BLOCK_ID(*idl, i),
 799                     (ID_BLOCK_NIDS(*idl) - i) * sizeof(ID) );
 800
 801         ID_BLOCK_ID(*idl, i) = id;
 802         ID_BLOCK_NIDS(*idl)++;
 803         (void) memset(
 804                 (char *) &ID_BLOCK_ID((*idl), ID_BLOCK_NIDS(*idl)),
 805                 '\0',
 806             (ID_BLOCK_NMAX(*idl) - ID_BLOCK_NIDS(*idl)) * sizeof(ID) );
 807
 808 #ifdef LDBM_DEBUG_IDL
 809         idl_check(*idl);
 810 #endif
 811
 812         return( i == 0 ? 1 : 0 );       /* inserted - first id changed or not */
 813 }
 814
 815
 816 int
 817 idl_delete_key (
 818         Backend         *be,
 819         DBCache  *db,
 820         Datum           key,
 821         ID              id
 822 )
 823 {
 824         Datum  data;
 825         ID_BLOCK *idl;
 826         unsigned i;
 827         int j, nids;
 828
 829         if ( (idl = idl_fetch_one( be, db, key ) ) == NULL )
 830         {
 831                 /* It wasn't found.  Hmm... */
 832                 return -1;
 833         }
 834
 835         if ( ID_BLOCK_ALLIDS( idl ) ) {
 836                 idl_free( idl );
 837                 return 0;
 838         }
 839
 840         if ( ! ID_BLOCK_INDIRECT( idl ) ) {
 841                 for ( i=0; i < ID_BLOCK_NIDS(idl); i++ ) {
 842                         if ( ID_BLOCK_ID(idl, i) == id ) {
 843                                 if( --ID_BLOCK_NIDS(idl) == 0 ) {
 844                                         ldbm_cache_delete( db, key );
 845
 846                                 } else {
 847                                         AC_MEMCPY(
 848                                                 &ID_BLOCK_ID(idl, i),
 849                                                 &ID_BLOCK_ID(idl, i+1),
 850                                                 (ID_BLOCK_NIDS(idl)-i) * sizeof(ID) );
 851
 852                                         ID_BLOCK_ID(idl, ID_BLOCK_NIDS(idl)) = NOID;
 853
 854                                         idl_store( be, db, key, idl );
 855                                 }
 856
 857                                 idl_free( idl );
 858                                 return 0;
 859                         }
 860                         /*  We didn't find the ID.  Hmmm... */
 861                 }
 862                 idl_free( idl );
 863                 return -1;
 864         }
 865
 866         /* We have to go through an indirect block and find the ID
 867            in the list of IDL's
 868            */
 869         for ( nids = 0; !ID_BLOCK_NOID(idl, nids); nids++ )
 870                 ;       /* NULL */
 871
 872         cont_alloc( &data, &key );
 873
 874         for ( j = 0; !ID_BLOCK_NOID(idl, j); j++ )
 875         {
 876                 ID_BLOCK *tmp;
 877                 cont_id( &data, ID_BLOCK_ID(idl, j) );
 878
 879                 if ( (tmp = idl_fetch_one( be, db, data )) == NULL ) {
 880 #ifdef NEW_LOGGING
 881                         LDAP_LOG(( "cache", LDAP_LEVEL_INFO,
 882                                    "idl_delete_key: idl_fetch_one returned NULL\n" ));
 883 #else
 884                         Debug( LDAP_DEBUG_ANY,
 885                             "idl_delete_key: idl_fetch of returned NULL\n", 0, 0, 0 );
 886 #endif
 887
 888                         continue;
 889                 }
 890                 /*
 891                    Now try to find the ID in tmp
 892                 */
 893                 for ( i=0; i < ID_BLOCK_NIDS(tmp); i++ )
 894                 {
 895                         if ( ID_BLOCK_ID(tmp, i) == id )
 896                         {
 897                                 AC_MEMCPY(
 898                                         &ID_BLOCK_ID(tmp, i),
 899                                         &ID_BLOCK_ID(tmp, i+1),
 900                                         (ID_BLOCK_NIDS(tmp)-(i+1)) * sizeof(ID));
 901                                 ID_BLOCK_ID(tmp, ID_BLOCK_NIDS(tmp)-1 ) = NOID;
 902                                 ID_BLOCK_NIDS(tmp)--;
 903
 904                                 if ( ID_BLOCK_NIDS(tmp) ) {
 905                                         idl_store ( be, db, data, tmp );
 906
 907                                 } else {
 908                                         ldbm_cache_delete( db, data );
 909                                         AC_MEMCPY(
 910                                                 &ID_BLOCK_ID(idl, j),
 911                                                 &ID_BLOCK_ID(idl, j+1),
 912                                                 (nids-(j+1)) * sizeof(ID));
 913                                         ID_BLOCK_ID(idl, nids-1) = NOID;
 914                                         nids--;
 915                                         if ( ! nids )
 916                                                 ldbm_cache_delete( db, key );
 917                                         else
 918                                                 idl_store( be, db, key, idl );
 919                                 }
 920                                 idl_free( tmp );
 921                                 cont_free( &data );
 922                                 idl_free( idl );
 923                                 return 0;
 924                         }
 925                 }
 926                 idl_free( tmp );
 927         }
 928
 929         cont_free( &data );
 930         idl_free( idl );
 931         return -1;
 932 }
 933
 934
 935 /* return a duplicate of a single ID_BLOCK */
 936 static ID_BLOCK *
 937 idl_dup( ID_BLOCK *idl )
 938 {
 939         ID_BLOCK        *new;
 940
 941         if ( idl == NULL ) {
 942                 return( NULL );
 943         }
 944
 945         new = idl_alloc( ID_BLOCK_NMAX(idl) );
 946
 947         AC_MEMCPY(
 948                 (char *) new,
 949                 (char *) idl,
 950                 (ID_BLOCK_NMAX(idl) + ID_BLOCK_IDS_OFFSET) * sizeof(ID) );
 951
 952 #ifdef LDBM_DEBUG_IDL
 953         idl_check(new);
 954 #endif
 955
 956         return( new );
 957 }
 958
 959
 960 /* return the smaller ID_BLOCK */
 961 static ID_BLOCK *
 962 idl_min( ID_BLOCK *a, ID_BLOCK *b )
 963 {
 964         return( ID_BLOCK_NIDS(a) > ID_BLOCK_NIDS(b) ? b : a );
 965 }
 966
 967
 968 /*
 969  * idl_intersection - return a intersection b
 970  */
 971 ID_BLOCK *
 972 idl_intersection(
 973     Backend     *be,
 974     ID_BLOCK    *a,
 975     ID_BLOCK    *b
 976 )
 977 {
 978         unsigned int    ai, bi, ni;
 979         ID_BLOCK                *n;
 980
 981         if ( a == NULL || b == NULL ) {
 982                 return( NULL );
 983         }
 984         if ( ID_BLOCK_ALLIDS( a ) ) {
 985                 return( idl_dup( b ) );
 986         }
 987         if ( ID_BLOCK_ALLIDS( b ) ) {
 988                 return( idl_dup( a ) );
 989         }
 990
 991         n = idl_dup( idl_min( a, b ) );
 992
 993 #ifdef LDBM_DEBUG_IDL
 994         idl_check(a);
 995         idl_check(b);
 996 #endif
 997
 998         for ( ni = 0, ai = 0, bi = 0; ai < ID_BLOCK_NIDS(a); ai++ ) {
 999                 for ( ;
1000                         bi < ID_BLOCK_NIDS(b) && ID_BLOCK_ID(b, bi) < ID_BLOCK_ID(a, ai);
1001                         bi++ )
1002                 {
1003                         ;       /* NULL */
1004                 }
1005
1006                 if ( bi == ID_BLOCK_NIDS(b) ) {
1007                         break;
1008                 }
1009
1010                 if ( ID_BLOCK_ID(b, bi) == ID_BLOCK_ID(a, ai) ) {
1011                         ID_BLOCK_ID(n, ni++) = ID_BLOCK_ID(a, ai);
1012                 }
1013         }
1014
1015         if ( ni == 0 ) {
1016                 idl_free( n );
1017                 return( NULL );
1018         }
1019         ID_BLOCK_NIDS(n) = ni;
1020
1021 #ifdef LDBM_DEBUG_IDL
1022         idl_check(n);
1023 #endif
1024
1025         return( n );
1026 }
1027
1028
1029 /*
1030  * idl_union - return a union b
1031  */
1032 ID_BLOCK *
1033 idl_union(
1034     Backend     *be,
1035     ID_BLOCK    *a,
1036     ID_BLOCK    *b
1037 )
1038 {
1039         unsigned int    ai, bi, ni;
1040         ID_BLOCK                *n;
1041
1042         if ( a == NULL ) {
1043                 return( idl_dup( b ) );
1044         }
1045         if ( b == NULL ) {
1046                 return( idl_dup( a ) );
1047         }
1048         if ( ID_BLOCK_ALLIDS( a ) || ID_BLOCK_ALLIDS( b ) ) {
1049                 return( idl_allids( be ) );
1050         }
1051
1052 #ifdef LDBM_DEBUG_IDL
1053         idl_check(a);
1054         idl_check(b);
1055 #endif
1056
1057         if ( ID_BLOCK_NIDS(b) < ID_BLOCK_NIDS(a) ) {
1058                 n = a;
1059                 a = b;
1060                 b = n;
1061         }
1062
1063         n = idl_alloc( ID_BLOCK_NIDS(a) + ID_BLOCK_NIDS(b) );
1064
1065         for ( ni = 0, ai = 0, bi = 0;
1066                 ai < ID_BLOCK_NIDS(a) && bi < ID_BLOCK_NIDS(b);
1067                 )
1068         {
1069                 if ( ID_BLOCK_ID(a, ai) < ID_BLOCK_ID(b, bi) ) {
1070                         ID_BLOCK_ID(n, ni++) = ID_BLOCK_ID(a, ai++);
1071
1072                 } else if ( ID_BLOCK_ID(b, bi) < ID_BLOCK_ID(a, ai) ) {
1073                         ID_BLOCK_ID(n, ni++) = ID_BLOCK_ID(b, bi++);
1074
1075                 } else {
1076                         ID_BLOCK_ID(n, ni++) = ID_BLOCK_ID(a, ai);
1077                         ai++, bi++;
1078                 }
1079         }
1080
1081         for ( ; ai < ID_BLOCK_NIDS(a); ai++ ) {
1082                 ID_BLOCK_ID(n, ni++) = ID_BLOCK_ID(a, ai);
1083         }
1084         for ( ; bi < ID_BLOCK_NIDS(b); bi++ ) {
1085                 ID_BLOCK_ID(n, ni++) = ID_BLOCK_ID(b, bi);
1086         }
1087         ID_BLOCK_NIDS(n) = ni;
1088
1089 #ifdef LDBM_DEBUG_IDL
1090         idl_check(n);
1091 #endif
1092
1093         return( n );
1094 }
1095
1096
1097 /*
1098  * idl_notin - return a intersection ~b (or a minus b)
1099  */
1100 ID_BLOCK *
1101 idl_notin(
1102     Backend     *be,
1103     ID_BLOCK    *a,
1104     ID_BLOCK    *b
1105 )
1106 {
1107         unsigned int    ni, ai, bi;
1108         ID_BLOCK                *n;
1109
1110         if ( a == NULL ) {
1111                 return( NULL );
1112         }
1113         if ( b == NULL || ID_BLOCK_ALLIDS( b )) {
1114                 return( idl_dup( a ) );
1115         }
1116
1117         if ( ID_BLOCK_ALLIDS( a ) ) {
1118                 n = idl_alloc( SLAPD_LDBM_MIN_MAXIDS );
1119                 ni = 0;
1120
1121                 for ( ai = 1, bi = 0;
1122                         ai < ID_BLOCK_NIDS(a) && ni < ID_BLOCK_NMAX(n) && bi < ID_BLOCK_NMAX(b);
1123                         ai++ )
1124                 {
1125                         if ( ID_BLOCK_ID(b, bi) == ai ) {
1126                                 bi++;
1127                         } else {
1128                                 ID_BLOCK_ID(n, ni++) = ai;
1129                         }
1130                 }
1131
1132                 for ( ; ai < ID_BLOCK_NIDS(a) && ni < ID_BLOCK_NMAX(n); ai++ ) {
1133                         ID_BLOCK_ID(n, ni++) = ai;
1134                 }
1135
1136                 if ( ni == ID_BLOCK_NMAX(n) ) {
1137                         idl_free( n );
1138                         return( idl_allids( be ) );
1139                 } else {
1140                         ID_BLOCK_NIDS(n) = ni;
1141                         return( n );
1142                 }
1143         }
1144
1145         n = idl_dup( a );
1146
1147         ni = 0;
1148         for ( ai = 0, bi = 0; ai < ID_BLOCK_NIDS(a); ai++ ) {
1149                 for ( ;
1150                         bi < ID_BLOCK_NIDS(b) && ID_BLOCK_ID(b, bi) < ID_BLOCK_ID(a, ai);
1151                     bi++ )
1152                 {
1153                         ;       /* NULL */
1154                 }
1155
1156                 if ( bi == ID_BLOCK_NIDS(b) ) {
1157                         break;
1158                 }
1159
1160                 if ( ID_BLOCK_ID(b, bi) != ID_BLOCK_ID(a, ai) ) {
1161                         ID_BLOCK_ID(n, ni++) = ID_BLOCK_ID(a, ai);
1162                 }
1163         }
1164
1165         for ( ; ai < ID_BLOCK_NIDS(a); ai++ ) {
1166                 ID_BLOCK_ID(n, ni++) = ID_BLOCK_ID(a, ai);
1167         }
1168         ID_BLOCK_NIDS(n) = ni;
1169
1170 #ifdef LDBM_DEBUG_IDL
1171         idl_check(n);
1172 #endif
1173
1174         return( n );
1175 }
1176
1177 /*      return the first ID in the block
1178  *      if ALLIDS block
1179  *              NIDS > 1 return 1
1180  *              otherwise return NOID
1181  *      otherwise return first ID
1182  *
1183  *      cursor is set to 1
1184  */
1185 ID
1186 idl_firstid( ID_BLOCK *idl, ID *cursor )
1187 {
1188         *cursor = 1;
1189
1190         if ( idl == NULL || ID_BLOCK_NIDS(idl) == 0 ) {
1191                 return( NOID );
1192         }
1193
1194         if ( ID_BLOCK_ALLIDS( idl ) ) {
1195                 return( ID_BLOCK_NIDS(idl) > 1 ? 1 : NOID );
1196         }
1197
1198         return( ID_BLOCK_ID(idl, 0) );
1199 }
1200
1201 /*      return next ID
1202  *      if ALLIDS block, cursor is id.
1203  *              increment id
1204  *              if id < NIDS return id
1205  *              otherwise NOID.
1206  *      otherwise cursor is index into block
1207  *              if index < nids
1208  *                      return id at index then increment
1209  */
1210 ID
1211 idl_nextid( ID_BLOCK *idl, ID *cursor )
1212 {
1213         if ( ID_BLOCK_ALLIDS( idl ) ) {
1214                 if( ++(*cursor) < ID_BLOCK_NIDS(idl) ) {
1215                         return *cursor;
1216                 } else {
1217                         return NOID;
1218                 }
1219         }
1220
1221         if ( *cursor < ID_BLOCK_NIDS(idl) ) {
1222                 return( ID_BLOCK_ID(idl, (*cursor)++) );
1223         }
1224
1225         return( NOID );
1226 }