From: Jong Hyuk Choi Date: Tue, 14 Dec 2004 17:22:22 +0000 (+0000) Subject: adaptive caching implementation X-Git-Tag: OPENLDAP_REL_ENG_2_3_0ALPHA~45 X-Git-Url: https://git.sur5r.net/?a=commitdiff_plain;h=273052b57356716b1baa11743ff6b7107e72907b;p=openldap adaptive caching implementation --- diff --git a/servers/slapd/back-bdb/back-bdb.h b/servers/slapd/back-bdb/back-bdb.h index 7b39a8e404..327862790e 100644 --- a/servers/slapd/back-bdb/back-bdb.h +++ b/servers/slapd/back-bdb/back-bdb.h @@ -114,6 +114,10 @@ typedef struct bdb_entry_info { #endif Entry *bei_e; Avlnode *bei_kids; +#ifdef SLAP_ZONE_ALLOC + struct bdb_info *bei_bdb; + int bei_zseq; +#endif ldap_pvt_thread_mutex_t bei_kids_mutex; struct bdb_entry_info *bei_lrunext; /* for cache lru list */ @@ -135,6 +139,9 @@ typedef struct bdb_cache { ldap_pvt_thread_rdwr_t c_rwlock; ldap_pvt_thread_mutex_t lru_mutex; u_int32_t c_locker; /* used by lru cleaner */ +#ifdef SLAP_ZONE_ALLOC + void *c_zctx; +#endif } Cache; #define CACHE_READ_LOCK 0 diff --git a/servers/slapd/back-bdb/cache.c b/servers/slapd/back-bdb/cache.c index eaabdf020d..1d9f4899bb 100644 --- a/servers/slapd/back-bdb/cache.c +++ b/servers/slapd/back-bdb/cache.c @@ -230,6 +230,9 @@ bdb_entryinfo_add_internal( #ifdef BDB_HIER ei2->bei_rdn = ei->bei_rdn; #endif +#ifdef SLAP_ZONE_ALLOC + ei2->bei_bdb = bdb; +#endif /* Add to cache ID tree */ if (avl_insert( &bdb->bi_cache.c_idtree, ei2, bdb_id_cmp, avl_dup_error )) { @@ -393,6 +396,9 @@ hdb_cache_find_parent( ein->bei_nrdn = ei.bei_nrdn; ein->bei_rdn = ei.bei_rdn; ein->bei_ckids = ei.bei_ckids; +#ifdef SLAP_ZONE_ALLOC + ein->bei_bdb = bdb; +#endif ei.bei_ckids = 0; /* This node is not fully connected yet */ @@ -530,27 +536,32 @@ bdb_cache_lru_add( elprev = elru->bei_lruprev; /* Too many probes, not enough idle, give up */ - if (i > 10) break; + if (i > 10) + break; /* If we can successfully writelock it, then * the object is idle. */ - if ( bdb_cache_entry_db_lock( bdb->bi_dbenv, bdb->bi_cache.c_locker, elru, 1, 1, - lockp ) == 0 ) { + if ( bdb_cache_entry_db_lock( bdb->bi_dbenv, + bdb->bi_cache.c_locker, elru, 1, 1, lockp ) == 0 ) { int stop = 0; /* If there's no entry, or this node is in * the process of linking into the cache, * or this node is being deleted, skip it. */ - if ( !elru->bei_e || (elru->bei_state & + if ( !elru->bei_e || (elru->bei_state & ( CACHE_ENTRY_NOT_LINKED | CACHE_ENTRY_DELETED ))) { bdb_cache_entry_db_unlock( bdb->bi_dbenv, lockp ); continue; } LRU_DELETE( &bdb->bi_cache, elru ); elru->bei_e->e_private = NULL; +#ifdef SLAP_ZONE_ALLOC + bdb_entry_return( bdb, elru->bei_e, elru->bei_zseq ); +#else bdb_entry_return( elru->bei_e ); +#endif elru->bei_e = NULL; ldap_pvt_thread_rdwr_wlock( &bdb->bi_cache.c_rwlock ); --bdb->bi_cache.c_cursize; @@ -606,6 +617,9 @@ bdb_cache_find_id( ei.bei_id = id; +#ifdef SLAP_ZONE_ALLOC + slap_zh_rlock(bdb->bi_cache.c_zctx); +#endif /* If we weren't given any info, see if we have it already cached */ if ( !*eip ) { again: ldap_pvt_thread_rdwr_rlock( &bdb->bi_cache.c_rwlock ); @@ -649,7 +663,11 @@ again: ldap_pvt_thread_rdwr_rlock( &bdb->bi_cache.c_rwlock ); &ep->e_nname, eip ); if ( *eip ) islocked = 1; if ( rc ) { +#ifdef SLAP_ZONE_ALLOC + bdb_entry_return( bdb, ep, (*eip)->bei_zseq ); +#else bdb_entry_return( ep ); +#endif ep = NULL; } } @@ -665,7 +683,15 @@ again: ldap_pvt_thread_rdwr_rlock( &bdb->bi_cache.c_rwlock ); rc = DB_NOTFOUND; } else { /* Make sure only one thread tries to load the entry */ -load1: if ( !(*eip)->bei_e && !((*eip)->bei_state & CACHE_ENTRY_LOADING)) { +load1: +#ifdef SLAP_ZONE_ALLOC + if ((*eip)->bei_e && !slap_zn_validate( + bdb->bi_cache.c_zctx, (*eip)->bei_e, (*eip)->bei_zseq)) { + (*eip)->bei_e = NULL; + (*eip)->bei_zseq = 0; + } +#endif + if ( !(*eip)->bei_e && !((*eip)->bei_state & CACHE_ENTRY_LOADING)) { load = 1; (*eip)->bei_state |= CACHE_ENTRY_LOADING; } @@ -711,6 +737,9 @@ load1: if ( !(*eip)->bei_e && !((*eip)->bei_state & CACHE_ENTRY_LOADING)) { bdb_fix_dn( ep, 0 ); #endif (*eip)->bei_e = ep; +#ifdef SLAP_ZONE_ALLOC + (*eip)->bei_zseq = *((ber_len_t *)ep - 2); +#endif ep = NULL; } (*eip)->bei_state ^= CACHE_ENTRY_LOADING; @@ -771,7 +800,11 @@ load1: if ( !(*eip)->bei_e && !((*eip)->bei_state & CACHE_ENTRY_LOADING)) { bdb_cache_entryinfo_unlock( *eip ); } if ( ep ) { +#ifdef SLAP_ZONE_ALLOC + bdb_entry_return( bdb, ep, (*eip)->bei_zseq ); +#else bdb_entry_return( ep ); +#endif } if ( rc == 0 ) { @@ -800,6 +833,12 @@ load1: if ( !(*eip)->bei_e && !((*eip)->bei_state & CACHE_ENTRY_LOADING)) { } } +#ifdef SLAP_ZONE_ALLOC + if (rc == 0 && (*eip)->bei_e) { + slap_zn_rlock(bdb->bi_cache.c_zctx, (*eip)->bei_e); + } + slap_zh_runlock(bdb->bi_cache.c_zctx); +#endif return rc; } @@ -865,7 +904,11 @@ bdb_cache_add( /* bdb_csn_commit can cause this when adding the database root entry */ if ( new->bei_e ) { new->bei_e->e_private = NULL; +#ifdef SLAP_ZONE_ALLOC + bdb_entry_return( bdb, new->bei_e, new->bei_zseq ); +#else bdb_entry_return( new->bei_e ); +#endif } new->bei_e = e; e->e_private = new; @@ -1043,7 +1086,11 @@ bdb_cache_delete_cleanup( { if ( ei->bei_e ) { ei->bei_e->e_private = NULL; +#ifdef SLAP_ZONE_ALLOC + bdb_entry_return( ei->bei_bdb, ei->bei_e, ei->bei_zseq ); +#else bdb_entry_return( ei->bei_e ); +#endif ei->bei_e = NULL; } @@ -1123,7 +1170,11 @@ bdb_entryinfo_release( void *data ) } if ( ei->bei_e ) { ei->bei_e->e_private = NULL; +#ifdef SLAP_ZONE_ALLOC + bdb_entry_return( ei->bei_bdb, ei->bei_e, ei->bei_zseq ); +#else bdb_entry_return( ei->bei_e ); +#endif } bdb_cache_entryinfo_destroy( ei ); } @@ -1310,7 +1361,11 @@ bdb_cache_delete_entry( if ( ei->bei_e && !(ei->bei_state & CACHE_ENTRY_NOT_LINKED )) { LRU_DELETE( &bdb->bi_cache, ei ); ei->bei_e->e_private = NULL; +#ifdef SLAP_ZONE_ALLOC + bdb_entry_return( bdb, ei->bei_e, ei->bei_zseq ); +#else bdb_entry_return( ei->bei_e ); +#endif ei->bei_e = NULL; --bdb->bi_cache.c_cursize; } diff --git a/servers/slapd/back-bdb/id2entry.c b/servers/slapd/back-bdb/id2entry.c index 1ef56e71d3..ba63ccea38 100644 --- a/servers/slapd/back-bdb/id2entry.c +++ b/servers/slapd/back-bdb/id2entry.c @@ -121,7 +121,11 @@ int bdb_id2entry( DBT2bv( &data, &bv ); - rc = entry_decode( &bv, e ); +#ifdef SLAP_ZONE_ALLOC + rc = entry_decode(&bv, e, bdb->bi_cache.c_zctx); +#else + rc = entry_decode(&bv, e); +#endif if( rc == 0 ) { (*e)->e_id = id; @@ -129,8 +133,13 @@ int bdb_id2entry( /* only free on error. On success, the entry was * decoded in place. */ - ch_free( data.data ); +#ifndef SLAP_ZONE_ALLOC + ch_free(data.data); +#endif } +#ifdef SLAP_ZONE_ALLOC + ch_free(data.data); +#endif return rc; } @@ -157,9 +166,23 @@ int bdb_id2entry_delete( return rc; } +#ifdef SLAP_ZONE_ALLOC int bdb_entry_return( - Entry *e ) + struct bdb_info *bdb, + Entry *e, + int zseq +) +#else +int bdb_entry_return( + Entry *e +) +#endif { +#ifdef SLAP_ZONE_ALLOC + if (!slap_zn_validate(bdb->bi_cache.c_zctx, e, zseq)) { + return 0; + } +#endif /* Our entries are allocated in two blocks; the data comes from * the db itself and the Entry structure and associated pointers * are allocated in entry_decode. The db data pointer is saved @@ -168,10 +191,11 @@ int bdb_entry_return( * is when an entry has been modified, in which case we also need * to free e_attrs. */ + #ifdef LDAP_COMP_MATCH comp_tree_free( e->e_attrs ); #endif - if( !e->e_bv.bv_val ) { /* A regular entry, from do_add */ + if( !e->e_bv.bv_val ) { /* Entry added by do_add */ entry_free( e ); return 0; } @@ -187,6 +211,7 @@ int bdb_entry_return( e->e_name.bv_val = NULL; e->e_nname.bv_val = NULL; } +#ifndef SLAP_ZONE_ALLOC #ifndef BDB_HIER /* In tool mode the e_bv buffer is realloc'd, leave it alone */ if( !(slapMode & SLAP_TOOL_MODE) ) { @@ -194,18 +219,24 @@ int bdb_entry_return( } #else free( e->e_bv.bv_val ); -#endif +#endif /* BDB_HIER */ +#endif /* !SLAP_ZONE_ALLOC */ + +#ifdef SLAP_ZONE_ALLOC + slap_zn_free( e, bdb->bi_cache.c_zctx ); +#else free( e ); +#endif return 0; } int bdb_entry_release( - Operation *o, + Operation *op, Entry *e, int rw ) { - struct bdb_info *bdb = (struct bdb_info *) o->o_bd->be_private; + struct bdb_info *bdb = (struct bdb_info *) op->o_bd->be_private; struct bdb_op_info *boi = NULL; /* slapMode : SLAP_SERVER_MODE, SLAP_TOOL_MODE, @@ -214,11 +245,15 @@ int bdb_entry_release( if ( slapMode == SLAP_SERVER_MODE ) { /* If not in our cache, just free it */ if ( !e->e_private ) { +#ifdef SLAP_ZONE_ALLOC + return bdb_entry_return( bdb, e, -1 ); +#else return bdb_entry_return( e ); +#endif } /* free entry and reader or writer lock */ - if ( o ) { - boi = (struct bdb_op_info *)o->o_private; + if ( op ) { + boi = (struct bdb_op_info *)op->o_private; } /* lock is freed with txn */ if ( !boi || boi->boi_txn ) { @@ -231,20 +266,32 @@ int bdb_entry_release( bdb_cache_return_entry_rw( bdb->bi_dbenv, &bdb->bi_cache, e, rw, &bli->bli_lock ); prev->bli_next = bli->bli_next; - o->o_tmpfree( bli, o->o_tmpmemctx ); + op->o_tmpfree( bli, op->o_tmpmemctx ); break; } } if ( !boi->boi_locks ) { - o->o_tmpfree( boi, o->o_tmpmemctx ); - o->o_private = NULL; + op->o_tmpfree( boi, op->o_tmpmemctx ); + op->o_private = NULL; } } } else { +#ifdef SLAP_ZONE_ALLOC + int zseq = -1; + if (e->e_private != NULL) { + BEI(e)->bei_e = NULL; + zseq = BEI(e)->bei_zseq; + } +#else if (e->e_private != NULL) BEI(e)->bei_e = NULL; +#endif e->e_private = NULL; +#ifdef SLAP_ZONE_ALLOC + bdb_entry_return ( bdb, e, zseq ); +#else bdb_entry_return ( e ); +#endif } return 0; diff --git a/servers/slapd/back-bdb/init.c b/servers/slapd/back-bdb/init.c index c8a351ca1d..6733592bf5 100644 --- a/servers/slapd/back-bdb/init.c +++ b/servers/slapd/back-bdb/init.c @@ -192,6 +192,16 @@ bdb_db_open( BackendDB *be ) /* One long-lived TXN per thread, two TXNs per write op */ bdb->bi_dbenv->set_tx_max( bdb->bi_dbenv, connection_pool_max * 3 ); +#ifdef SLAP_ZONE_ALLOC + if ( bdb->bi_cache.c_maxsize ) { + bdb->bi_cache.c_zctx = slap_zn_mem_create( + SLAP_ZONE_INITSIZE, + SLAP_ZONE_MAXSIZE, + SLAP_ZONE_DELTA, + SLAP_ZONE_SIZE); + } +#endif + if ( bdb->bi_idl_cache_max_size ) { bdb->bi_idl_tree = NULL; ldap_pvt_thread_rdwr_init( &bdb->bi_idl_tree_rwlock ); diff --git a/servers/slapd/back-bdb/proto-bdb.h b/servers/slapd/back-bdb/proto-bdb.h index 7cc4916478..8480db421e 100644 --- a/servers/slapd/back-bdb/proto-bdb.h +++ b/servers/slapd/back-bdb/proto-bdb.h @@ -189,11 +189,14 @@ int bdb_id2entry_delete( DB_TXN *tid, Entry *e); +#ifdef SLAP_ZONE_ALLOC +#else int bdb_id2entry( BackendDB *be, DB_TXN *tid, ID id, Entry **e); +#endif #define bdb_entry_free BDB_SYMBOL(entry_free) #define bdb_entry_return BDB_SYMBOL(entry_return) @@ -201,7 +204,11 @@ int bdb_id2entry( #define bdb_entry_get BDB_SYMBOL(entry_get) void bdb_entry_free ( Entry *e ); +#ifdef SLAP_ZONE_ALLOC +int bdb_entry_return( struct bdb_info *bdb, Entry *e, int seqno ); +#else int bdb_entry_return( Entry *e ); +#endif BI_entry_release_rw bdb_entry_release; BI_entry_get_rw bdb_entry_get; @@ -427,7 +434,7 @@ void bdb_unlocked_cache_return_entry_rw( Cache *cache, Entry *e, int rw ); #define bdb_cache_add BDB_SYMBOL(cache_add) #define bdb_cache_children BDB_SYMBOL(cache_children) #define bdb_cache_delete BDB_SYMBOL(cache_delete) -#define bdb_cache_delete_cleanup BDB_SYMBOL(cache_delete_cleanup) +#define bdb_cache_delete_cleanup BDB_SYMBOL(cache_delete_cleanup) #define bdb_cache_find_id BDB_SYMBOL(cache_find_id) #define bdb_cache_find_info BDB_SYMBOL(cache_find_info) #define bdb_cache_find_ndn BDB_SYMBOL(cache_find_ndn) diff --git a/servers/slapd/back-bdb/search.c b/servers/slapd/back-bdb/search.c index 0c99ffddbc..e47ce1f86b 100644 --- a/servers/slapd/back-bdb/search.c +++ b/servers/slapd/back-bdb/search.c @@ -842,6 +842,9 @@ dn2entry_retry: ? get_entry_referrals( op, matched ) : NULL; +#ifdef SLAP_ZONE_ALLOC + slap_zn_runlock(bdb->bi_cache.c_zctx, matched); +#endif bdb_cache_return_entry_r (bdb->bi_dbenv, &bdb->bi_cache, matched, &lock); matched = NULL; @@ -853,6 +856,9 @@ dn2entry_retry: } } else { +#ifdef SLAP_ZONE_ALLOC + slap_zn_runlock(bdb->bi_cache.c_zctx, matched); +#endif rs->sr_ref = referral_rewrite( default_referral, NULL, &sop->o_req_dn, sop->oq_search.rs_scope ); } @@ -882,6 +888,9 @@ dn2entry_retry: ber_dupbv( &matched_dn, &e->e_name ); erefs = get_entry_referrals( op, e ); +#ifdef SLAP_ZONE_ALLOC + slap_zn_runlock(bdb->bi_cache.c_zctx, e); +#endif bdb_cache_return_entry_r( bdb->bi_dbenv, &bdb->bi_cache, e, &lock ); e = NULL; @@ -913,6 +922,9 @@ dn2entry_retry: ( test_filter( op, e, get_assertion( op )) != LDAP_COMPARE_TRUE )) { rs->sr_err = LDAP_ASSERTION_FAILED; +#ifdef SLAP_ZONE_ALLOC + slap_zn_runlock(bdb->bi_cache.c_zctx, e); +#endif send_ldap_result( sop, rs ); return 1; } @@ -930,6 +942,9 @@ dn2entry_retry: base.e_nname = realbase; base.e_id = e->e_id; +#ifdef SLAP_ZONE_ALLOC + slap_zn_runlock(bdb->bi_cache.c_zctx, e); +#endif if ( e != &e_root ) { bdb_cache_return_entry_r(bdb->bi_dbenv, &bdb->bi_cache, e, &lock); } @@ -1230,7 +1245,7 @@ loop_begin: #ifdef BDB_PSEARCH if (!IS_PSEARCH) { #endif -id2entry_retry: +fetch_entry_retry: /* get the entry with reader lock */ ei = NULL; rs->sr_err = bdb_cache_find_id( op, ltid, @@ -1244,7 +1259,7 @@ id2entry_retry: } else if ( rs->sr_err == DB_LOCK_DEADLOCK || rs->sr_err == DB_LOCK_NOTGRANTED ) { - goto id2entry_retry; + goto fetch_entry_retry; } if ( ei && rs->sr_err == LDAP_SUCCESS ) { @@ -1471,6 +1486,9 @@ id2entry_retry: ) { #ifdef BDB_PSEARCH if (!IS_PSEARCH) { +#endif +#ifdef SLAP_ZONE_ALLOC + slap_zn_runlock(bdb->bi_cache.c_zctx, e); #endif bdb_cache_return_entry_r( bdb->bi_dbenv, &bdb->bi_cache, e, &lock ); @@ -1670,6 +1688,9 @@ post_search_no_entry: case -1: /* connection closed */ #ifdef BDB_PSEARCH if (!IS_PSEARCH) { +#endif +#ifdef SLAP_ZONE_ALLOC + slap_zn_runlock(bdb->bi_cache.c_zctx, e); #endif bdb_cache_return_entry_r(bdb->bi_dbenv, &bdb->bi_cache, e, &lock); @@ -1697,6 +1718,9 @@ loop_continue: if (!IS_PSEARCH) { if (!(IS_POST_SEARCH && entry_sync_state == LDAP_SYNC_DELETE)) { +#ifdef SLAP_ZONE_ALLOC + slap_zn_runlock(bdb->bi_cache.c_zctx, e); +#endif bdb_cache_return_entry_r( bdb->bi_dbenv, &bdb->bi_cache, e , &lock ); if ( sop->o_nocaching ) { @@ -1705,6 +1729,9 @@ loop_continue: } } #else +#ifdef SLAP_ZONE_ALLOC + slap_zn_runlock(bdb->bi_cache.c_zctx, e); +#endif bdb_cache_return_entry_r( bdb->bi_dbenv, &bdb->bi_cache, e , &lock ); #endif @@ -1867,6 +1894,9 @@ done: if( !IS_PSEARCH && e != NULL ) { /* free reader lock */ +#ifdef SLAP_ZONE_ALLOC + slap_zn_runlock(bdb->bi_cache.c_zctx, e); +#endif bdb_cache_return_entry_r( bdb->bi_dbenv, &bdb->bi_cache, e, &lock ); } ber_bvfree( search_context_csn ); diff --git a/servers/slapd/back-bdb/tools.c b/servers/slapd/back-bdb/tools.c index d76aef4dc1..033b0bd8bd 100644 --- a/servers/slapd/back-bdb/tools.c +++ b/servers/slapd/back-bdb/tools.c @@ -176,7 +176,12 @@ Entry* bdb_tool_entry_get( BackendDB *be, ID id ) #ifndef BDB_HIER DBT2bv( &data, &bv ); +#ifdef SLAP_ZONE_ALLOC + /* FIXME: will add ctx later */ + rc = entry_decode( &bv, &e, NULL ); +#else rc = entry_decode( &bv, &e ); +#endif if( rc == LDAP_SUCCESS ) { e->e_id = id;