+
+#ifdef BDB_TOOL_IDL_CACHING
+static int
+bdb_tool_idl_cmp( const void *v1, const void *v2 )
+{
+ const bdb_tool_idl_cache *c1 = v1, *c2 = v2;
+ int rc;
+
+ if (( rc = c1->kstr.bv_len - c2->kstr.bv_len )) return rc;
+ return memcmp( c1->kstr.bv_val, c2->kstr.bv_val, c1->kstr.bv_len );
+}
+
+static int
+bdb_tool_idl_flush_one( void *v1, void *arg )
+{
+ bdb_tool_idl_cache *ic = v1;
+ DB *db = arg;
+ struct bdb_info *bdb = bdb_tool_info;
+ bdb_tool_idl_cache_entry *ice;
+ DBC *curs;
+ DBT key, data;
+ int i, rc;
+ ID id, nid;
+
+ /* Freshly allocated, ignore it */
+ if ( !ic->head && ic->count <= BDB_IDL_DB_SIZE ) {
+ return 0;
+ }
+
+ rc = db->cursor( db, NULL, &curs, 0 );
+ if ( rc )
+ return -1;
+
+ DBTzero( &key );
+ DBTzero( &data );
+
+ bv2DBT( &ic->kstr, &key );
+
+ data.size = data.ulen = sizeof( ID );
+ data.flags = DB_DBT_USERMEM;
+ data.data = &nid;
+
+ rc = curs->c_get( curs, &key, &data, DB_SET );
+ /* If key already exists and we're writing a range... */
+ if ( rc == 0 && ic->count > BDB_IDL_DB_SIZE ) {
+ /* If it's not currently a range, must delete old info */
+ if ( nid ) {
+ /* Skip lo */
+ while ( curs->c_get( curs, &key, &data, DB_NEXT_DUP ) == 0 )
+ curs->c_del( curs, 0 );
+
+ nid = 0;
+ /* Store range marker */
+ curs->c_put( curs, &key, &data, DB_KEYFIRST );
+ } else {
+
+ /* Skip lo */
+ rc = curs->c_get( curs, &key, &data, DB_NEXT_DUP );
+
+ /* Get hi */
+ rc = curs->c_get( curs, &key, &data, DB_NEXT_DUP );
+
+ /* Delete hi */
+ curs->c_del( curs, 0 );
+ }
+ BDB_ID2DISK( ic->last, &nid );
+ curs->c_put( curs, &key, &data, DB_KEYLAST );
+ rc = 0;
+ } else if ( rc && rc != DB_NOTFOUND ) {
+ rc = -1;
+ } else if ( ic->count > BDB_IDL_DB_SIZE ) {
+ /* range, didn't exist before */
+ nid = 0;
+ rc = curs->c_put( curs, &key, &data, DB_KEYLAST );
+ if ( rc == 0 ) {
+ BDB_ID2DISK( ic->first, &nid );
+ rc = curs->c_put( curs, &key, &data, DB_KEYLAST );
+ if ( rc == 0 ) {
+ BDB_ID2DISK( ic->last, &nid );
+ rc = curs->c_put( curs, &key, &data, DB_KEYLAST );
+ }
+ }
+ if ( rc ) {
+ rc = -1;
+ }
+ } else {
+ int n;
+
+ /* Just a normal write */
+ rc = 0;
+ for ( ice = ic->head, n=0; ice; ice = ice->next, n++ ) {
+ int end;
+ if ( ice->next ) {
+ end = IDBLOCK;
+ } else {
+ end = ic->count & (IDBLOCK-1);
+ if ( !end )
+ end = IDBLOCK;
+ }
+ for ( i=0; i<end; i++ ) {
+ if ( !ice->ids[i] ) continue;
+ BDB_ID2DISK( ice->ids[i], &nid );
+ rc = curs->c_put( curs, &key, &data, DB_NODUPDATA );
+ if ( rc ) {
+ if ( rc == DB_KEYEXIST ) {
+ rc = 0;
+ continue;
+ }
+ rc = -1;
+ break;
+ }
+ }
+ if ( rc ) {
+ rc = -1;
+ break;
+ }
+ }
+ if ( ic->head ) {
+ ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
+ ic->tail->next = bdb_tool_idl_free_list;
+ bdb_tool_idl_free_list = ic->head;
+ bdb->bi_idl_cache_size -= n;
+ ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
+ }
+ }
+ if ( ic != db->app_private ) {
+ ch_free( ic );
+ } else {
+ ic->head = ic->tail = NULL;
+ }
+ curs->c_close( curs );
+ return rc;
+}
+
+static int
+bdb_tool_idl_flush_db( DB *db, bdb_tool_idl_cache *ic )
+{
+ Avlnode *root = db->app_private;
+ int rc;
+
+ db->app_private = ic;
+ rc = avl_apply( root, bdb_tool_idl_flush_one, db, -1, AVL_INORDER );
+ avl_free( root, NULL );
+ db->app_private = NULL;
+ if ( rc != -1 )
+ rc = 0;
+ return rc;
+}
+
+static int
+bdb_tool_idl_flush( BackendDB *be )
+{
+ struct bdb_info *bdb = (struct bdb_info *) be->be_private;
+ DB *db;
+ Avlnode *root;
+ int i, rc = 0;
+
+ for ( i=BDB_NDB; i < bdb->bi_ndatabases; i++ ) {
+ db = bdb->bi_databases[i]->bdi_db;
+ if ( !db->app_private ) continue;
+ rc = bdb_tool_idl_flush_db( db, NULL );
+ if ( rc )
+ break;
+ }
+ if ( !rc ) {
+ bdb->bi_idl_cache_size = 0;
+ }
+ return rc;
+}
+
+int bdb_tool_idl_add(
+ BackendDB *be,
+ DB *db,
+ DB_TXN *txn,
+ DBT *key,
+ ID id )
+{
+ struct bdb_info *bdb = (struct bdb_info *) be->be_private;
+ bdb_tool_idl_cache *ic, itmp;
+ bdb_tool_idl_cache_entry *ice;
+ int rc;
+
+ if ( !bdb->bi_idl_cache_max_size )
+ return bdb_idl_insert_key( be, db, txn, key, id );
+
+ DBT2bv( key, &itmp.kstr );
+
+ ic = avl_find( (Avlnode *)db->app_private, &itmp, bdb_tool_idl_cmp );
+
+ /* No entry yet, create one */
+ if ( !ic ) {
+ DBC *curs;
+ DBT data;
+ ID nid;
+ int rc;
+
+ ic = ch_malloc( sizeof( bdb_tool_idl_cache ) + itmp.kstr.bv_len );
+ ic->kstr.bv_len = itmp.kstr.bv_len;
+ ic->kstr.bv_val = (char *)(ic+1);
+ AC_MEMCPY( ic->kstr.bv_val, itmp.kstr.bv_val, ic->kstr.bv_len );
+ ic->head = ic->tail = NULL;
+ ic->last = 0;
+ ic->count = 0;
+ avl_insert( (Avlnode **)&db->app_private, ic, bdb_tool_idl_cmp,
+ avl_dup_error );
+
+ /* load existing key count here */
+ rc = db->cursor( db, NULL, &curs, 0 );
+ if ( rc ) return rc;
+
+ data.ulen = sizeof( ID );
+ data.flags = DB_DBT_USERMEM;
+ data.data = &nid;
+ rc = curs->c_get( curs, key, &data, DB_SET );
+ if ( rc == 0 ) {
+ if ( nid == 0 ) {
+ ic->count = BDB_IDL_DB_SIZE+1;
+ } else {
+ db_recno_t count;
+
+ curs->c_count( curs, &count, 0 );
+ ic->count = count;
+ BDB_DISK2ID( &nid, &ic->first );
+ }
+ }
+ curs->c_close( curs );
+ }
+ /* are we a range already? */
+ if ( ic->count > BDB_IDL_DB_SIZE ) {
+ ic->last = id;
+ return 0;
+ /* Are we at the limit, and converting to a range? */
+ } else if ( ic->count == BDB_IDL_DB_SIZE ) {
+ int n;
+ for ( ice = ic->head, n=0; ice; ice = ice->next, n++ )
+ /* counting */ ;
+ if ( n ) {
+ ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
+ ic->tail->next = bdb_tool_idl_free_list;
+ bdb_tool_idl_free_list = ic->head;
+ bdb->bi_idl_cache_size -= n;
+ ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
+ }
+ ic->head = ic->tail = NULL;
+ ic->last = id;
+ ic->count++;
+ return 0;
+ }
+ /* No free block, create that too */
+ if ( !ic->tail || ( ic->count & (IDBLOCK-1)) == 0) {
+ ice = NULL;
+ ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
+ if ( bdb->bi_idl_cache_size >= bdb->bi_idl_cache_max_size ) {
+ ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
+ rc = bdb_tool_idl_flush_db( db, ic );
+ if ( rc )
+ return rc;
+ avl_insert( (Avlnode **)&db->app_private, ic, bdb_tool_idl_cmp,
+ avl_dup_error );
+ ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock );
+ }
+ bdb->bi_idl_cache_size++;
+ if ( bdb_tool_idl_free_list ) {
+ ice = bdb_tool_idl_free_list;
+ bdb_tool_idl_free_list = ice->next;
+ }
+ ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock );
+ if ( !ice ) {
+ ice = ch_malloc( sizeof( bdb_tool_idl_cache_entry ));
+ }
+ memset( ice, 0, sizeof( *ice ));
+ if ( !ic->head ) {
+ ic->head = ice;
+ } else {
+ ic->tail->next = ice;
+ }
+ ic->tail = ice;
+ if ( !ic->count )
+ ic->first = id;
+ }
+ ice = ic->tail;
+ ice->ids[ ic->count & (IDBLOCK-1) ] = id;
+ ic->count++;
+
+ return 0;
+}
+#endif
+
+static void *
+bdb_tool_trickle_task( void *ctx, void *ptr )
+{
+ DB_ENV *env = ptr;
+ int wrote;
+
+ ldap_pvt_thread_mutex_lock( &bdb_tool_trickle_mutex );
+ while ( 1 ) {
+ ldap_pvt_thread_cond_wait( &bdb_tool_trickle_cond,
+ &bdb_tool_trickle_mutex );
+ if ( slapd_shutdown )
+ break;
+ env->memp_trickle( env, 30, &wrote );
+ }
+ ldap_pvt_thread_mutex_unlock( &bdb_tool_trickle_mutex );
+}
+
+static void *
+bdb_tool_index_task( void *ctx, void *ptr )
+{
+ int base = *(int *)ptr;
+
+ free( ptr );
+ while ( 1 ) {
+ ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex );
+ bdb_tool_index_tcount--;
+ if ( !bdb_tool_index_tcount )
+ ldap_pvt_thread_cond_signal( &bdb_tool_index_cond_main );
+ ldap_pvt_thread_cond_wait( &bdb_tool_index_cond_work,
+ &bdb_tool_index_mutex );
+ ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex );
+ if ( slapd_shutdown )
+ break;
+
+ bdb_tool_index_threads[base] = bdb_index_recrun( bdb_tool_ix_op,
+ bdb_tool_info, bdb_tool_index_rec, bdb_tool_ix_id, base );
+ }
+
+ return NULL;
+}