git.sur5r.net Git - openldap/blob - servers/slapd/back-bdb/idl.c

   1 /* idl.c - ldap id list handling routines */
   2 /* $OpenLDAP$ */
   3 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
   4  *
   5  * Copyright 2000-2005 The OpenLDAP Foundation.
   6  * All rights reserved.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted only as authorized by the OpenLDAP
  10  * Public License.
  11  *
  12  * A copy of this license is available in the file LICENSE in the
  13  * top-level directory of the distribution or, alternatively, at
  14  * <http://www.OpenLDAP.org/license.html>.
  15  */
  16
  17 #include "portable.h"
  18
  19 #include <stdio.h>
  20 #include <ac/string.h>
  21
  22 #include "back-bdb.h"
  23 #include "idl.h"
  24
  25 #define IDL_MAX(x,y)    ( x > y ? x : y )
  26 #define IDL_MIN(x,y)    ( x < y ? x : y )
  27
  28 #define IDL_CMP(x,y)    ( x < y ? -1 : ( x > y ? 1 : 0 ) )
  29
  30 #define IDL_LRU_DELETE( bdb, e ) do {                                   \
  31         if ( e->idl_lru_prev != NULL ) {                                \
  32                 e->idl_lru_prev->idl_lru_next = e->idl_lru_next;        \
  33         } else {                                                        \
  34                 bdb->bi_idl_lru_head = e->idl_lru_next;                 \
  35         }                                                               \
  36         if ( e->idl_lru_next != NULL ) {                                \
  37                 e->idl_lru_next->idl_lru_prev = e->idl_lru_prev;        \
  38         } else {                                                        \
  39                 bdb->bi_idl_lru_tail = e->idl_lru_prev;                 \
  40         }                                                               \
  41 } while ( 0 )
  42
  43 #define IDL_LRU_ADD( bdb, e ) do {                                      \
  44         e->idl_lru_next = bdb->bi_idl_lru_head;                         \
  45         if ( e->idl_lru_next != NULL ) {                                \
  46                 e->idl_lru_next->idl_lru_prev = (e);                    \
  47         }                                                               \
  48         (bdb)->bi_idl_lru_head = (e);                                   \
  49         e->idl_lru_prev = NULL;                                         \
  50         if ( (bdb)->bi_idl_lru_tail == NULL ) {                         \
  51                 (bdb)->bi_idl_lru_tail = (e);                           \
  52         }                                                               \
  53 } while ( 0 )
  54
  55 static int
  56 bdb_idl_entry_cmp( const void *v_idl1, const void *v_idl2 )
  57 {
  58         const bdb_idl_cache_entry_t *idl1 = v_idl1, *idl2 = v_idl2;
  59         int rc;
  60
  61         if ((rc = SLAP_PTRCMP( idl1->db, idl2->db ))) return rc;
  62         if ((rc = idl1->kstr.bv_len - idl2->kstr.bv_len )) return rc;
  63         return ( memcmp ( idl1->kstr.bv_val, idl2->kstr.bv_val , idl1->kstr.bv_len ) );
  64 }
  65
  66 #if IDL_DEBUG > 0
  67 static void idl_check( ID *ids )
  68 {
  69         if( BDB_IDL_IS_RANGE( ids ) ) {
  70                 assert( BDB_IDL_RANGE_FIRST(ids) <= BDB_IDL_RANGE_LAST(ids) );
  71         } else {
  72                 ID i;
  73                 for( i=1; i < ids[0]; i++ ) {
  74                         assert( ids[i+1] > ids[i] );
  75                 }
  76         }
  77 }
  78
  79 #if IDL_DEBUG > 1
  80 static void idl_dump( ID *ids )
  81 {
  82         if( BDB_IDL_IS_RANGE( ids ) ) {
  83                 Debug( LDAP_DEBUG_ANY,
  84                         "IDL: range ( %ld - %ld )\n",
  85                         (long) BDB_IDL_RANGE_FIRST( ids ),
  86                         (long) BDB_IDL_RANGE_LAST( ids ) );
  87
  88         } else {
  89                 ID i;
  90                 Debug( LDAP_DEBUG_ANY, "IDL: size %ld", (long) ids[0], 0, 0 );
  91
  92                 for( i=1; i<=ids[0]; i++ ) {
  93                         if( i % 16 == 1 ) {
  94                                 Debug( LDAP_DEBUG_ANY, "\n", 0, 0, 0 );
  95                         }
  96                         Debug( LDAP_DEBUG_ANY, "  %02lx", (long) ids[i], 0, 0 );
  97                 }
  98
  99                 Debug( LDAP_DEBUG_ANY, "\n", 0, 0, 0 );
 100         }
 101
 102         idl_check( ids );
 103 }
 104 #endif /* IDL_DEBUG > 1 */
 105 #endif /* IDL_DEBUG > 0 */
 106
 107 unsigned bdb_idl_search( ID *ids, ID id )
 108 {
 109 #define IDL_BINARY_SEARCH 1
 110 #ifdef IDL_BINARY_SEARCH
 111         /*
 112          * binary search of id in ids
 113          * if found, returns position of id
 114          * if not found, returns first postion greater than id
 115          */
 116         unsigned base = 0;
 117         unsigned cursor = 0;
 118         int val = 0;
 119         unsigned n = ids[0];
 120
 121 #if IDL_DEBUG > 0
 122         idl_check( ids );
 123 #endif
 124
 125         while( 0 < n ) {
 126                 int pivot = n >> 1;
 127                 cursor = base + pivot;
 128                 val = IDL_CMP( id, ids[cursor + 1] );
 129
 130                 if( val < 0 ) {
 131                         n = pivot;
 132
 133                 } else if ( val > 0 ) {
 134                         base = cursor + 1;
 135                         n -= pivot + 1;
 136
 137                 } else {
 138                         return cursor + 1;
 139                 }
 140         }
 141
 142         if( val > 0 ) {
 143                 return cursor + 2;
 144         } else {
 145                 return cursor + 1;
 146         }
 147
 148 #else
 149         /* (reverse) linear search */
 150         int i;
 151
 152 #if IDL_DEBUG > 0
 153         idl_check( ids );
 154 #endif
 155
 156         for( i=ids[0]; i; i-- ) {
 157                 if( id > ids[i] ) {
 158                         break;
 159                 }
 160         }
 161
 162         return i+1;
 163 #endif
 164 }
 165
 166 int bdb_idl_insert( ID *ids, ID id )
 167 {
 168         unsigned x;
 169
 170 #if IDL_DEBUG > 1
 171         Debug( LDAP_DEBUG_ANY, "insert: %04lx at %d\n", (long) id, x, 0 );
 172         idl_dump( ids );
 173 #elif IDL_DEBUG > 0
 174         idl_check( ids );
 175 #endif
 176
 177         if (BDB_IDL_IS_RANGE( ids )) {
 178                 /* if already in range, treat as a dup */
 179                 if (id >= BDB_IDL_FIRST(ids) && id <= BDB_IDL_LAST(ids))
 180                         return -1;
 181                 if (id < BDB_IDL_FIRST(ids))
 182                         ids[1] = id;
 183                 else if (id > BDB_IDL_LAST(ids))
 184                         ids[2] = id;
 185                 return 0;
 186         }
 187
 188         x = bdb_idl_search( ids, id );
 189         assert( x > 0 );
 190
 191         if( x < 1 ) {
 192                 /* internal error */
 193                 return -2;
 194         }
 195
 196         if ( x <= ids[0] && ids[x] == id ) {
 197                 /* duplicate */
 198                 return -1;
 199         }
 200
 201         if ( ++ids[0] >= BDB_IDL_DB_MAX ) {
 202                 if( id < ids[1] ) {
 203                         ids[1] = id;
 204                         ids[2] = ids[ids[0]-1];
 205                 } else if ( ids[ids[0]-1] < id ) {
 206                         ids[2] = id;
 207                 } else {
 208                         ids[2] = ids[ids[0]-1];
 209                 }
 210                 ids[0] = NOID;
 211
 212         } else {
 213                 /* insert id */
 214                 AC_MEMCPY( &ids[x+1], &ids[x], (ids[0]-x) * sizeof(ID) );
 215                 ids[x] = id;
 216         }
 217
 218 #if IDL_DEBUG > 1
 219         idl_dump( ids );
 220 #elif IDL_DEBUG > 0
 221         idl_check( ids );
 222 #endif
 223
 224         return 0;
 225 }
 226
 227 #if 0   /* unused */
 228 static int idl_delete( ID *ids, ID id )
 229 {
 230         unsigned x = bdb_idl_search( ids, id );
 231
 232 #if IDL_DEBUG > 1
 233         Debug( LDAP_DEBUG_ANY, "delete: %04lx at %d\n", (long) id, x, 0 );
 234         idl_dump( ids );
 235 #elif IDL_DEBUG > 0
 236         idl_check( ids );
 237 #endif
 238
 239         assert( x > 0 );
 240
 241         if( x <= 0 ) {
 242                 /* internal error */
 243                 return -2;
 244         }
 245
 246         if( x > ids[0] || ids[x] != id ) {
 247                 /* not found */
 248                 return -1;
 249
 250         } else if ( --ids[0] == 0 ) {
 251                 if( x != 1 ) {
 252                         return -3;
 253                 }
 254
 255         } else {
 256                 AC_MEMCPY( &ids[x], &ids[x+1], (1+ids[0]-x) * sizeof(ID) );
 257         }
 258
 259 #if IDL_DEBUG > 1
 260         idl_dump( ids );
 261 #elif IDL_DEBUG > 0
 262         idl_check( ids );
 263 #endif
 264
 265         return 0;
 266 }
 267 #endif  /* unused */
 268
 269 static char *
 270 bdb_show_key(
 271         DBT             *key,
 272         char            *buf )
 273 {
 274         if ( key->size == sizeof( ID ) ) {
 275                 unsigned char *c = key->data;
 276                 sprintf( buf, "[%02x%02x%02x%02x]", c[0], c[1], c[2], c[3] );
 277                 return buf;
 278         } else {
 279                 return key->data;
 280         }
 281 }
 282
 283 /* Find a db/key pair in the IDL cache. If ids is non-NULL,
 284  * copy the cached IDL into it, otherwise just return the status.
 285  */
 286 int
 287 bdb_idl_cache_get(
 288         struct bdb_info *bdb,
 289         DB                      *db,
 290         DBT                     *key,
 291         ID                      *ids )
 292 {
 293         bdb_idl_cache_entry_t idl_tmp;
 294         bdb_idl_cache_entry_t *matched_idl_entry;
 295
 296         DBT2bv( key, &idl_tmp.kstr );
 297         idl_tmp.db = db;
 298         ldap_pvt_thread_rdwr_rlock( &bdb->bi_idl_tree_rwlock );
 299         matched_idl_entry = avl_find( bdb->bi_idl_tree, &idl_tmp,
 300                                       bdb_idl_entry_cmp );
 301         if ( matched_idl_entry != NULL ) {
 302                 if ( matched_idl_entry->idl && ids )
 303                         BDB_IDL_CPY( ids, matched_idl_entry->idl );
 304                 ldap_pvt_thread_rdwr_runlock( &bdb->bi_idl_tree_rwlock );
 305                 ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
 306                 IDL_LRU_DELETE( bdb, matched_idl_entry );
 307                 IDL_LRU_ADD( bdb, matched_idl_entry );
 308                 ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
 309                 if ( matched_idl_entry->idl )
 310                         return LDAP_SUCCESS;
 311                 else
 312                         return DB_NOTFOUND;
 313         }
 314         ldap_pvt_thread_rdwr_runlock( &bdb->bi_idl_tree_rwlock );
 315
 316         return LDAP_NO_SUCH_OBJECT;
 317 }
 318
 319 void
 320 bdb_idl_cache_put(
 321         struct bdb_info *bdb,
 322         DB                      *db,
 323         DBT                     *key,
 324         ID                      *ids,
 325         int                     rc )
 326 {
 327         bdb_idl_cache_entry_t idl_tmp;
 328         bdb_idl_cache_entry_t *ee;
 329
 330         DBT2bv( key, &idl_tmp.kstr );
 331
 332         ee = (bdb_idl_cache_entry_t *) ch_malloc(
 333                 sizeof( bdb_idl_cache_entry_t ) );
 334         ee->db = db;
 335         if ( rc == DB_NOTFOUND) {
 336                 ee->idl = NULL;
 337         } else {
 338                 ee->idl = (ID*) ch_malloc( BDB_IDL_SIZEOF ( ids ) );
 339                 BDB_IDL_CPY( ee->idl, ids );
 340         }
 341         ee->idl_lru_prev = NULL;
 342         ee->idl_lru_next = NULL;
 343         ber_dupbv( &ee->kstr, &idl_tmp.kstr );
 344         ldap_pvt_thread_rdwr_wlock( &bdb->bi_idl_tree_rwlock );
 345         if ( avl_insert( &bdb->bi_idl_tree, (caddr_t) ee,
 346                 bdb_idl_entry_cmp, avl_dup_error ))
 347         {
 348                 ch_free( ee->kstr.bv_val );
 349                 ch_free( ee->idl );
 350                 ch_free( ee );
 351                 ldap_pvt_thread_rdwr_wunlock( &bdb->bi_idl_tree_rwlock );
 352                 return;
 353         }
 354         ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
 355         IDL_LRU_ADD( bdb, ee );
 356         if ( ++bdb->bi_idl_cache_size > bdb->bi_idl_cache_max_size ) {
 357                 int i = 0;
 358                 while ( bdb->bi_idl_lru_tail != NULL && i < 10 ) {
 359                         ee = bdb->bi_idl_lru_tail;
 360                         if ( avl_delete( &bdb->bi_idl_tree, (caddr_t) ee,
 361                                     bdb_idl_entry_cmp ) == NULL ) {
 362                                 Debug( LDAP_DEBUG_ANY, "=> bdb_idl_cache_put: "
 363                                         "AVL delete failed\n",
 364                                         0, 0, 0 );
 365                         }
 366                         IDL_LRU_DELETE( bdb, ee );
 367                         i++;
 368                         --bdb->bi_idl_cache_size;
 369                         ch_free( ee->kstr.bv_val );
 370                         ch_free( ee->idl );
 371                         ch_free( ee );
 372                 }
 373         }
 374
 375         ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
 376         ldap_pvt_thread_rdwr_wunlock( &bdb->bi_idl_tree_rwlock );
 377 }
 378
 379 void
 380 bdb_idl_cache_del(
 381         struct bdb_info *bdb,
 382         DB                      *db,
 383         DBT                     *key )
 384 {
 385         bdb_idl_cache_entry_t *matched_idl_entry, idl_tmp;
 386         DBT2bv( key, &idl_tmp.kstr );
 387         idl_tmp.db = db;
 388         ldap_pvt_thread_rdwr_wlock( &bdb->bi_idl_tree_rwlock );
 389         matched_idl_entry = avl_find( bdb->bi_idl_tree, &idl_tmp,
 390                                       bdb_idl_entry_cmp );
 391         if ( matched_idl_entry != NULL ) {
 392                 if ( avl_delete( &bdb->bi_idl_tree, (caddr_t) matched_idl_entry,
 393                                     bdb_idl_entry_cmp ) == NULL ) {
 394                         Debug( LDAP_DEBUG_ANY, "=> bdb_idl_cache_del: "
 395                                 "AVL delete failed\n",
 396                                 0, 0, 0 );
 397                 }
 398                 --bdb->bi_idl_cache_size;
 399                 ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
 400                 IDL_LRU_DELETE( bdb, matched_idl_entry );
 401                 ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
 402                 free( matched_idl_entry->kstr.bv_val );
 403                 if ( matched_idl_entry->idl )
 404                         free( matched_idl_entry->idl );
 405                 free( matched_idl_entry );
 406         }
 407         ldap_pvt_thread_rdwr_wunlock( &bdb->bi_idl_tree_rwlock );
 408 }
 409
 410 int
 411 bdb_idl_fetch_key(
 412         BackendDB       *be,
 413         DB                      *db,
 414         DB_TXN          *tid,
 415         DBT                     *key,
 416         ID                      *ids,
 417         DBC                     **saved_cursor,
 418         int                     get_flag )
 419 {
 420         struct bdb_info *bdb = (struct bdb_info *) be->be_private;
 421         int rc;
 422         DBT data, key2, *kptr;
 423         DBC *cursor;
 424         ID *i;
 425         void *ptr;
 426         size_t len;
 427         int rc2;
 428         int flags = bdb->bi_db_opflags | DB_MULTIPLE;
 429         int opflag;
 430
 431         /* If using BerkeleyDB 4.0, the buf must be large enough to
 432          * grab the entire IDL in one get(), otherwise BDB will leak
 433          * resources on subsequent get's.  We can safely call get()
 434          * twice - once for the data, and once to get the DB_NOTFOUND
 435          * result meaning there's no more data. See ITS#2040 for details.
 436          * This bug is fixed in BDB 4.1 so a smaller buffer will work if
 437          * stack space is too limited.
 438          *
 439          * configure now requires Berkeley DB 4.1.
 440          */
 441 #if DB_VERSION_FULL < 0x04010000
 442 #       define BDB_ENOUGH 5
 443 #else
 444         /* We sometimes test with tiny IDLs, and BDB always wants buffers
 445          * that are at least one page in size.
 446          */
 447 # if BDB_IDL_DB_SIZE < 4096
 448 #   define BDB_ENOUGH 2048
 449 # else
 450 #       define BDB_ENOUGH 1
 451 # endif
 452 #endif
 453         ID buf[BDB_IDL_DB_SIZE*BDB_ENOUGH];
 454
 455         char keybuf[16];
 456
 457         Debug( LDAP_DEBUG_ARGS,
 458                 "bdb_idl_fetch_key: %s\n",
 459                 bdb_show_key( key, keybuf ), 0, 0 );
 460
 461         assert( ids != NULL );
 462
 463         if ( saved_cursor && *saved_cursor ) {
 464                 opflag = DB_NEXT;
 465         } else if ( get_flag == LDAP_FILTER_GE ) {
 466                 opflag = DB_SET_RANGE;
 467         } else if ( get_flag == LDAP_FILTER_LE ) {
 468                 opflag = DB_FIRST;
 469         } else {
 470                 opflag = DB_SET;
 471         }
 472
 473         /* only non-range lookups can use the IDL cache */
 474         if ( bdb->bi_idl_cache_size && opflag == DB_SET ) {
 475                 rc = bdb_idl_cache_get( bdb, db, key, ids );
 476                 if ( rc != LDAP_NO_SUCH_OBJECT ) return rc;
 477         }
 478
 479         DBTzero( &data );
 480
 481         data.data = buf;
 482         data.ulen = sizeof(buf);
 483         data.flags = DB_DBT_USERMEM;
 484
 485         if ( tid ) flags |= DB_RMW;
 486
 487         /* If we're not reusing an existing cursor, get a new one */
 488         if( opflag != DB_NEXT ) {
 489                 rc = db->cursor( db, tid, &cursor, bdb->bi_db_opflags );
 490                 if( rc != 0 ) {
 491                         Debug( LDAP_DEBUG_ANY, "=> bdb_idl_fetch_key: "
 492                                 "cursor failed: %s (%d)\n", db_strerror(rc), rc, 0 );
 493                         return rc;
 494                 }
 495         } else {
 496                 cursor = *saved_cursor;
 497         }
 498
 499         /* If this is a LE lookup, save original key so we can determine
 500          * when to stop. If this is a GE lookup, save the key since it
 501          * will be overwritten.
 502          */
 503         if ( get_flag == LDAP_FILTER_LE || get_flag == LDAP_FILTER_GE ) {
 504                 DBTzero( &key2 );
 505                 key2.flags = DB_DBT_USERMEM;
 506                 key2.ulen = sizeof(keybuf);
 507                 key2.data = keybuf;
 508                 key2.size = key->size;
 509                 AC_MEMCPY( keybuf, key->data, key->size );
 510                 kptr = &key2;
 511         } else {
 512                 kptr = key;
 513         }
 514         len = key->size;
 515         rc = cursor->c_get( cursor, kptr, &data, flags | opflag );
 516
 517         /* skip presence key on range inequality lookups */
 518         while (rc == 0 && kptr->size != len) {
 519                 rc = cursor->c_get( cursor, kptr, &data, flags | DB_NEXT_NODUP );
 520         }
 521         /* If we're doing a LE compare and the new key is greater than
 522          * our search key, we're done
 523          */
 524         if (rc == 0 && get_flag == LDAP_FILTER_LE && memcmp( kptr->data,
 525                 key->data, key->size ) > 0 ) {
 526                 rc = DB_NOTFOUND;
 527         }
 528         if (rc == 0) {
 529                 i = ids;
 530                 while (rc == 0) {
 531                         u_int8_t *j;
 532
 533                         DB_MULTIPLE_INIT( ptr, &data );
 534                         while (ptr) {
 535                                 DB_MULTIPLE_NEXT(ptr, &data, j, len);
 536                                 if (j) {
 537                                         ++i;
 538                                         BDB_DISK2ID( j, i );
 539                                 }
 540                         }
 541                         rc = cursor->c_get( cursor, key, &data, flags | DB_NEXT_DUP );
 542                 }
 543                 if ( rc == DB_NOTFOUND ) rc = 0;
 544                 ids[0] = i - ids;
 545                 /* On disk, a range is denoted by 0 in the first element */
 546                 if (ids[1] == 0) {
 547                         if (ids[0] != BDB_IDL_RANGE_SIZE) {
 548                                 Debug( LDAP_DEBUG_ANY, "=> bdb_idl_fetch_key: "
 549                                         "range size mismatch: expected %d, got %ld\n",
 550                                         BDB_IDL_RANGE_SIZE, ids[0], 0 );
 551                                 cursor->c_close( cursor );
 552                                 return -1;
 553                         }
 554                         BDB_IDL_RANGE( ids, ids[2], ids[3] );
 555                 }
 556                 data.size = BDB_IDL_SIZEOF(ids);
 557         }
 558
 559         if ( saved_cursor && rc == 0 ) {
 560                 if ( !*saved_cursor )
 561                         *saved_cursor = cursor;
 562                 rc2 = 0;
 563         }
 564         else
 565                 rc2 = cursor->c_close( cursor );
 566         if (rc2) {
 567                 Debug( LDAP_DEBUG_ANY, "=> bdb_idl_fetch_key: "
 568                         "close failed: %s (%d)\n", db_strerror(rc2), rc2, 0 );
 569                 return rc2;
 570         }
 571
 572         if( rc == DB_NOTFOUND ) {
 573                 return rc;
 574
 575         } else if( rc != 0 ) {
 576                 Debug( LDAP_DEBUG_ANY, "=> bdb_idl_fetch_key: "
 577                         "get failed: %s (%d)\n",
 578                         db_strerror(rc), rc, 0 );
 579                 return rc;
 580
 581         } else if ( data.size == 0 || data.size % sizeof( ID ) ) {
 582                 /* size not multiple of ID size */
 583                 Debug( LDAP_DEBUG_ANY, "=> bdb_idl_fetch_key: "
 584                         "odd size: expected %ld multiple, got %ld\n",
 585                         (long) sizeof( ID ), (long) data.size, 0 );
 586                 return -1;
 587
 588         } else if ( data.size != BDB_IDL_SIZEOF(ids) ) {
 589                 /* size mismatch */
 590                 Debug( LDAP_DEBUG_ANY, "=> bdb_idl_fetch_key: "
 591                         "get size mismatch: expected %ld, got %ld\n",
 592                         (long) ((1 + ids[0]) * sizeof( ID )), (long) data.size, 0 );
 593                 return -1;
 594         }
 595
 596         if ( bdb->bi_idl_cache_max_size ) {
 597                 bdb_idl_cache_put( bdb, db, key, ids, rc );
 598         }
 599
 600         return rc;
 601 }
 602
 603
 604 int
 605 bdb_idl_insert_key(
 606         BackendDB       *be,
 607         DB                      *db,
 608         DB_TXN          *tid,
 609         DBT                     *key,
 610         ID                      id )
 611 {
 612         struct bdb_info *bdb = (struct bdb_info *) be->be_private;
 613         int     rc;
 614         DBT data;
 615         DBC *cursor;
 616         ID lo, hi, tmp, nlo, nhi, nid;
 617         char *err;
 618
 619         {
 620                 char buf[16];
 621                 Debug( LDAP_DEBUG_ARGS,
 622                         "bdb_idl_insert_key: %lx %s\n",
 623                         (long) id, bdb_show_key( key, buf ), 0 );
 624         }
 625
 626         assert( id != NOID );
 627
 628         if ( bdb->bi_idl_cache_size ) {
 629                 bdb_idl_cache_del( bdb, db, key );
 630         }
 631
 632         DBTzero( &data );
 633         data.size = sizeof( ID );
 634         data.ulen = data.size;
 635         data.flags = DB_DBT_USERMEM;
 636
 637         BDB_ID2DISK( id, &nid );
 638
 639         rc = db->cursor( db, tid, &cursor, bdb->bi_db_opflags );
 640         if ( rc != 0 ) {
 641                 Debug( LDAP_DEBUG_ANY, "=> bdb_idl_insert_key: "
 642                         "cursor failed: %s (%d)\n", db_strerror(rc), rc, 0 );
 643                 return rc;
 644         }
 645         data.data = &nlo;
 646         /* Fetch the first data item for this key, to see if it
 647          * exists and if it's a range.
 648          */
 649         rc = cursor->c_get( cursor, key, &data, DB_SET | DB_RMW );
 650         err = "c_get";
 651         if ( rc == 0 ) {
 652                 if ( nlo != 0 ) {
 653                         /* not a range, count the number of items */
 654                         db_recno_t count;
 655                         rc = cursor->c_count( cursor, &count, 0 );
 656                         if ( rc != 0 ) {
 657                                 err = "c_count";
 658                                 goto fail;
 659                         }
 660                         if ( count >= BDB_IDL_DB_MAX ) {
 661                         /* No room, convert to a range */
 662                                 DBT key2 = *key;
 663                                 db_recno_t i;
 664
 665                                 key2.dlen = key2.ulen;
 666                                 key2.flags |= DB_DBT_PARTIAL;
 667
 668                                 BDB_DISK2ID( &nlo, &lo );
 669                                 data.data = &nhi;
 670
 671                                 rc = cursor->c_get( cursor, &key2, &data, DB_NEXT_NODUP );
 672                                 if ( rc != 0 && rc != DB_NOTFOUND ) {
 673                                         err = "c_get next_nodup";
 674                                         goto fail;
 675                                 }
 676                                 if ( rc == DB_NOTFOUND ) {
 677                                         rc = cursor->c_get( cursor, key, &data, DB_LAST );
 678                                         if ( rc != 0 ) {
 679                                                 err = "c_get last";
 680                                                 goto fail;
 681                                         }
 682                                 } else {
 683                                         rc = cursor->c_get( cursor, key, &data, DB_PREV );
 684                                         if ( rc != 0 ) {
 685                                                 err = "c_get prev";
 686                                                 goto fail;
 687                                         }
 688                                 }
 689                                 BDB_DISK2ID( &nhi, &hi );
 690                                 /* Update hi/lo if needed, then delete all the items
 691                                  * between lo and hi
 692                                  */
 693                                 data.data = &nid;
 694                                 if ( id > hi ) {
 695                                         rc = cursor->c_del( cursor, 0 );
 696                                         if ( rc != 0 ) {
 697                                                 err = "c_del hi";
 698                                                 goto fail;
 699                                         }
 700                                         rc = cursor->c_put( cursor, key, &data, DB_KEYLAST );
 701                                         if ( rc != 0 ) {
 702                                                 err = "c_put hi";
 703                                                 goto fail;
 704                                         }
 705                                 }
 706                                 /* Don't fetch anything, just position cursor */
 707                                 data.flags = DB_DBT_USERMEM | DB_DBT_PARTIAL;
 708                                 data.dlen = data.ulen = 0;
 709                                 rc = cursor->c_get( cursor, key, &data, DB_SET | DB_RMW );
 710                                 if ( rc != 0 ) {
 711                                         err = "c_get 2";
 712                                         goto fail;
 713                                 }
 714                                 if ( id < lo ) {
 715                                         rc = cursor->c_del( cursor, 0 );
 716                                         if ( rc != 0 ) {
 717                                                 err = "c_del lo";
 718                                                 goto fail;
 719                                         }
 720                                         rc = cursor->c_put( cursor, key, &data, DB_KEYFIRST );
 721                                         if ( rc != 0 ) {
 722                                                 err = "c_put lo";
 723                                                 goto fail;
 724                                         }
 725                                 }
 726                                 /* Delete all the records between lo and hi */
 727                                 for ( i=2; i<count; i++ ) {
 728                                         rc = cursor->c_get( cursor, &key2, &data, DB_NEXT_DUP | DB_RMW );
 729                                         if ( rc != 0 ) {
 730                                                 err = "c_get next_dup";
 731                                                 goto fail;
 732                                         }
 733                                         rc = cursor->c_del( cursor, 0 );
 734                                         if ( rc != 0 ) {
 735                                                 err = "c_del range";
 736                                                 goto fail;
 737                                         }
 738                                 }
 739                                 /* Store the range marker */
 740                                 data.size = data.ulen = sizeof(ID);
 741                                 data.flags = DB_DBT_USERMEM;
 742                                 nid = 0;
 743                                 rc = cursor->c_put( cursor, key, &data, DB_KEYFIRST );
 744                                 if ( rc != 0 ) {
 745                                         err = "c_put range";
 746                                         goto fail;
 747                                 }
 748                         } else {
 749                         /* There's room, just store it */
 750                                 goto put1;
 751                         }
 752                 } else {
 753                         /* It's a range, see if we need to rewrite
 754                          * the boundaries
 755                          */
 756                         hi = id;
 757                         data.data = &nlo;
 758                         rc = cursor->c_get( cursor, key, &data, DB_NEXT_DUP );
 759                         if ( rc != 0 ) {
 760                                 err = "c_get lo";
 761                                 goto fail;
 762                         }
 763                         BDB_DISK2ID( &nlo, &lo );
 764                         if ( id > lo ) {
 765                                 data.data = &nhi;
 766                                 rc = cursor->c_get( cursor, key, &data, DB_NEXT_DUP );
 767                                 if ( rc != 0 ) {
 768                                         err = "c_get hi";
 769                                         goto fail;
 770                                 }
 771                                 BDB_DISK2ID( &nhi, &hi );
 772                         }
 773                         if ( id < lo || id > hi ) {
 774                                 /* Delete the current lo/hi */
 775                                 rc = cursor->c_del( cursor, 0 );
 776                                 if ( rc != 0 ) {
 777                                         err = "c_del";
 778                                         goto fail;
 779                                 }
 780                                 data.data = &nid;
 781                                 rc = cursor->c_put( cursor, key, &data, DB_KEYFIRST );
 782                                 if ( rc != 0 ) {
 783                                         err = "c_put lo/hi";
 784                                         goto fail;
 785                                 }
 786                         }
 787                 }
 788         } else if ( rc == DB_NOTFOUND ) {
 789 put1:           data.data = &nid;
 790                 rc = cursor->c_put( cursor, key, &data, DB_NODUPDATA );
 791                 /* Don't worry if it's already there */
 792                 if ( rc != 0 && rc != DB_KEYEXIST ) {
 793                         err = "c_put id";
 794                         goto fail;
 795                 }
 796         } else {
 797                 /* initial c_get failed, nothing was done */
 798 fail:
 799                 Debug( LDAP_DEBUG_ANY, "=> bdb_idl_insert_key: "
 800                         "%s failed: %s (%d)\n", err, db_strerror(rc), rc );
 801                 cursor->c_close( cursor );
 802                 return rc;
 803         }
 804         rc = cursor->c_close( cursor );
 805         if( rc != 0 ) {
 806                 Debug( LDAP_DEBUG_ANY, "=> bdb_idl_insert_key: "
 807                         "c_close failed: %s (%d)\n",
 808                         db_strerror(rc), rc, 0 );
 809         }
 810         return rc;
 811 }
 812
 813 int
 814 bdb_idl_delete_key(
 815         BackendDB       *be,
 816         DB                      *db,
 817         DB_TXN          *tid,
 818         DBT                     *key,
 819         ID                      id )
 820 {
 821         struct bdb_info *bdb = (struct bdb_info *) be->be_private;
 822         int     rc;
 823         DBT data;
 824         DBC *cursor;
 825         ID lo, hi, tmp, nid, nlo, nhi;
 826         char *err;
 827
 828         {
 829                 char buf[16];
 830                 Debug( LDAP_DEBUG_ARGS,
 831                         "bdb_idl_delete_key: %lx %s\n",
 832                         (long) id, bdb_show_key( key, buf ), 0 );
 833         }
 834         assert( id != NOID );
 835
 836         if ( bdb->bi_idl_cache_max_size ) {
 837                 bdb_idl_cache_del( bdb, db, key );
 838         }
 839
 840         BDB_ID2DISK( id, &nid );
 841
 842         DBTzero( &data );
 843         data.data = &tmp;
 844         data.size = sizeof( id );
 845         data.ulen = data.size;
 846         data.flags = DB_DBT_USERMEM;
 847
 848         rc = db->cursor( db, tid, &cursor, bdb->bi_db_opflags );
 849         if ( rc != 0 ) {
 850                 Debug( LDAP_DEBUG_ANY, "=> bdb_idl_delete_key: "
 851                         "cursor failed: %s (%d)\n", db_strerror(rc), rc, 0 );
 852                 return rc;
 853         }
 854         /* Fetch the first data item for this key, to see if it
 855          * exists and if it's a range.
 856          */
 857         rc = cursor->c_get( cursor, key, &data, DB_SET | DB_RMW );
 858         err = "c_get";
 859         if ( rc == 0 ) {
 860                 if ( tmp != 0 ) {
 861                         /* Not a range, just delete it */
 862                         if (tmp != nid) {
 863                                 /* position to correct item */
 864                                 tmp = nid;
 865                                 rc = cursor->c_get( cursor, key, &data,
 866                                         DB_GET_BOTH | DB_RMW  );
 867                                 if ( rc != 0 ) {
 868                                         err = "c_get id";
 869                                         goto fail;
 870                                 }
 871                         }
 872                         rc = cursor->c_del( cursor, 0 );
 873                         if ( rc != 0 ) {
 874                                 err = "c_del id";
 875                                 goto fail;
 876                         }
 877                 } else {
 878                         /* It's a range, see if we need to rewrite
 879                          * the boundaries
 880                          */
 881                         data.data = &nlo;
 882                         rc = cursor->c_get( cursor, key, &data, DB_NEXT_DUP );
 883                         if ( rc != 0 ) {
 884                                 err = "c_get lo";
 885                                 goto fail;
 886                         }
 887                         BDB_DISK2ID( &nlo, &lo );
 888                         data.data = &nhi;
 889                         rc = cursor->c_get( cursor, key, &data, DB_NEXT_DUP );
 890                         if ( rc != 0 ) {
 891                                 err = "c_get hi";
 892                                 goto fail;
 893                         }
 894                         BDB_DISK2ID( &nhi, &hi );
 895                         if ( id == lo || id == hi ) {
 896                                 if ( id == lo ) {
 897                                         id++;
 898                                         lo = id;
 899                                 } else if ( id == hi ) {
 900                                         id--;
 901                                         hi = id;
 902                                 }
 903                                 if ( lo >= hi ) {
 904                                 /* The range has collapsed... */
 905                                         rc = db->del( db, tid, key, 0 );
 906                                         if ( rc != 0 ) {
 907                                                 err = "del";
 908                                                 goto fail;
 909                                         }
 910                                 } else {
 911                                         if ( id == lo ) {
 912                                                 /* reposition on lo slot */
 913                                                 data.data = &nlo;
 914                                                 cursor->c_get( cursor, key, &data, DB_PREV );
 915                                         }
 916                                         rc = cursor->c_del( cursor, 0 );
 917                                         if ( rc != 0 ) {
 918                                                 err = "c_del";
 919                                                 goto fail;
 920                                         }
 921                                 }
 922                                 if ( lo <= hi ) {
 923                                         BDB_ID2DISK( id, &nid );
 924                                         data.data = &nid;
 925                                         rc = cursor->c_put( cursor, key, &data, DB_KEYFIRST );
 926                                         if ( rc != 0 ) {
 927                                                 err = "c_put lo/hi";
 928                                                 goto fail;
 929                                         }
 930                                 }
 931                         }
 932                 }
 933         } else {
 934                 /* initial c_get failed, nothing was done */
 935 fail:
 936                 if ( rc != DB_NOTFOUND ) {
 937                 Debug( LDAP_DEBUG_ANY, "=> bdb_idl_delete_key: "
 938                         "%s failed: %s (%d)\n", err, db_strerror(rc), rc );
 939                 }
 940                 cursor->c_close( cursor );
 941                 return rc;
 942         }
 943         rc = cursor->c_close( cursor );
 944         if( rc != 0 ) {
 945                 Debug( LDAP_DEBUG_ANY,
 946                         "=> bdb_idl_delete_key: c_close failed: %s (%d)\n",
 947                         db_strerror(rc), rc, 0 );
 948         }
 949
 950         return rc;
 951 }
 952
 953
 954 /*
 955  * idl_intersection - return a = a intersection b
 956  */
 957 int
 958 bdb_idl_intersection(
 959         ID *a,
 960         ID *b )
 961 {
 962         ID ida, idb;
 963         ID idmax, idmin;
 964         ID cursora = 0, cursorb = 0, cursorc;
 965         int swap = 0;
 966
 967         if ( BDB_IDL_IS_ZERO( a ) || BDB_IDL_IS_ZERO( b ) ) {
 968                 a[0] = 0;
 969                 return 0;
 970         }
 971
 972         idmin = IDL_MAX( BDB_IDL_FIRST(a), BDB_IDL_FIRST(b) );
 973         idmax = IDL_MIN( BDB_IDL_LAST(a), BDB_IDL_LAST(b) );
 974         if ( idmin > idmax ) {
 975                 a[0] = 0;
 976                 return 0;
 977         } else if ( idmin == idmax ) {
 978                 a[0] = 1;
 979                 a[1] = idmin;
 980                 return 0;
 981         }
 982
 983         if ( BDB_IDL_IS_RANGE( a ) ) {
 984                 if ( BDB_IDL_IS_RANGE(b) ) {
 985                 /* If both are ranges, just shrink the boundaries */
 986                         a[1] = idmin;
 987                         a[2] = idmax;
 988                         return 0;
 989                 } else {
 990                 /* Else swap so that b is the range, a is a list */
 991                         ID *tmp = a;
 992                         a = b;
 993                         b = tmp;
 994                         swap = 1;
 995                 }
 996         }
 997
 998         /* If a range completely covers the list, the result is
 999          * just the list. If idmin to idmax is contiguous, just
1000          * turn it into a range.
1001          */
1002         if ( BDB_IDL_IS_RANGE( b )
1003                 && BDB_IDL_FIRST( b ) <= BDB_IDL_FIRST( a )
1004                 && BDB_IDL_LAST( b ) >= BDB_IDL_LAST( a ) ) {
1005                 if (idmax - idmin + 1 == a[0])
1006                 {
1007                         a[0] = NOID;
1008                         a[1] = idmin;
1009                         a[2] = idmax;
1010                 }
1011                 goto done;
1012         }
1013
1014         /* Fine, do the intersection one element at a time.
1015          * First advance to idmin in both IDLs.
1016          */
1017         cursora = cursorb = idmin;
1018         ida = bdb_idl_first( a, &cursora );
1019         idb = bdb_idl_first( b, &cursorb );
1020         cursorc = 0;
1021
1022         while( ida <= idmax || idb <= idmax ) {
1023                 if( ida == idb ) {
1024                         a[++cursorc] = ida;
1025                         ida = bdb_idl_next( a, &cursora );
1026                         idb = bdb_idl_next( b, &cursorb );
1027                 } else if ( ida < idb ) {
1028                         ida = bdb_idl_next( a, &cursora );
1029                 } else {
1030                         idb = bdb_idl_next( b, &cursorb );
1031                 }
1032         }
1033         a[0] = cursorc;
1034 done:
1035         if (swap)
1036                 BDB_IDL_CPY( b, a );
1037
1038         return 0;
1039 }
1040
1041
1042 /*
1043  * idl_union - return a = a union b
1044  */
1045 int
1046 bdb_idl_union(
1047         ID      *a,
1048         ID      *b )
1049 {
1050         ID ida, idb;
1051         ID cursora = 0, cursorb = 0, cursorc;
1052
1053         if ( BDB_IDL_IS_ZERO( b ) ) {
1054                 return 0;
1055         }
1056
1057         if ( BDB_IDL_IS_ZERO( a ) ) {
1058                 BDB_IDL_CPY( a, b );
1059                 return 0;
1060         }
1061
1062         if ( BDB_IDL_IS_RANGE( a ) || BDB_IDL_IS_RANGE(b) ) {
1063 over:           ida = IDL_MIN( BDB_IDL_FIRST(a), BDB_IDL_FIRST(b) );
1064                 idb = IDL_MAX( BDB_IDL_LAST(a), BDB_IDL_LAST(b) );
1065                 a[0] = NOID;
1066                 a[1] = ida;
1067                 a[2] = idb;
1068                 return 0;
1069         }
1070
1071         ida = bdb_idl_first( a, &cursora );
1072         idb = bdb_idl_first( b, &cursorb );
1073
1074         cursorc = b[0];
1075
1076         /* The distinct elements of a are cat'd to b */
1077         while( ida != NOID || idb != NOID ) {
1078                 if ( ida < idb ) {
1079                         if( ++cursorc > BDB_IDL_UM_MAX ) {
1080                                 goto over;
1081                         }
1082                         b[cursorc] = ida;
1083                         ida = bdb_idl_next( a, &cursora );
1084
1085                 } else {
1086                         if ( ida == idb )
1087                                 ida = bdb_idl_next( a, &cursora );
1088                         idb = bdb_idl_next( b, &cursorb );
1089                 }
1090         }
1091
1092         /* b is copied back to a in sorted order */
1093         a[0] = cursorc;
1094         cursora = 1;
1095         cursorb = 1;
1096         cursorc = b[0]+1;
1097         while (cursorb <= b[0] || cursorc <= a[0]) {
1098                 if (cursorc > a[0])
1099                         idb = NOID;
1100                 else
1101                         idb = b[cursorc];
1102                 if (cursorb <= b[0] && b[cursorb] < idb)
1103                         a[cursora++] = b[cursorb++];
1104                 else {
1105                         a[cursora++] = idb;
1106                         cursorc++;
1107                 }
1108         }
1109
1110         return 0;
1111 }
1112
1113
1114 #if 0
1115 /*
1116  * bdb_idl_notin - return a intersection ~b (or a minus b)
1117  */
1118 int
1119 bdb_idl_notin(
1120         ID      *a,
1121         ID      *b,
1122         ID *ids )
1123 {
1124         ID ida, idb;
1125         ID cursora = 0, cursorb = 0;
1126
1127         if( BDB_IDL_IS_ZERO( a ) ||
1128                 BDB_IDL_IS_ZERO( b ) ||
1129                 BDB_IDL_IS_RANGE( b ) )
1130         {
1131                 BDB_IDL_CPY( ids, a );
1132                 return 0;
1133         }
1134
1135         if( BDB_IDL_IS_RANGE( a ) ) {
1136                 BDB_IDL_CPY( ids, a );
1137                 return 0;
1138         }
1139
1140         ida = bdb_idl_first( a, &cursora ),
1141         idb = bdb_idl_first( b, &cursorb );
1142
1143         ids[0] = 0;
1144
1145         while( ida != NOID ) {
1146                 if ( idb == NOID ) {
1147                         /* we could shortcut this */
1148                         ids[++ids[0]] = ida;
1149                         ida = bdb_idl_next( a, &cursora );
1150
1151                 } else if ( ida < idb ) {
1152                         ids[++ids[0]] = ida;
1153                         ida = bdb_idl_next( a, &cursora );
1154
1155                 } else if ( ida > idb ) {
1156                         idb = bdb_idl_next( b, &cursorb );
1157
1158                 } else {
1159                         ida = bdb_idl_next( a, &cursora );
1160                         idb = bdb_idl_next( b, &cursorb );
1161                 }
1162         }
1163
1164         return 0;
1165 }
1166 #endif
1167
1168 ID bdb_idl_first( ID *ids, ID *cursor )
1169 {
1170         ID pos;
1171
1172         if ( ids[0] == 0 ) {
1173                 *cursor = NOID;
1174                 return NOID;
1175         }
1176
1177         if ( BDB_IDL_IS_RANGE( ids ) ) {
1178                 if( *cursor < ids[1] ) {
1179                         *cursor = ids[1];
1180                 }
1181                 return *cursor;
1182         }
1183
1184         if ( *cursor == 0 )
1185                 pos = 1;
1186         else
1187                 pos = bdb_idl_search( ids, *cursor );
1188
1189         if( pos > ids[0] ) {
1190                 return NOID;
1191         }
1192
1193         *cursor = pos;
1194         return ids[pos];
1195 }
1196
1197 ID bdb_idl_next( ID *ids, ID *cursor )
1198 {
1199         if ( BDB_IDL_IS_RANGE( ids ) ) {
1200                 if( ids[2] < ++(*cursor) ) {
1201                         return NOID;
1202                 }
1203                 return *cursor;
1204         }
1205
1206         if ( ++(*cursor) <= ids[0] ) {
1207                 return ids[*cursor];
1208         }
1209
1210         return NOID;
1211 }
1212