adaptive caching implementation

diff --git a/servers/slapd/back-bdb/back-bdb.h b/servers/slapd/back-bdb/back-bdb.h
index 7b39a8e4046734a53dc92c64f10048cb8638de76..327862790ee9729a17c27b855b811cb8ce67beec 100644 (file)
--- a/servers/slapd/back-bdb/back-bdb.h
+++ b/servers/slapd/back-bdb/back-bdb.h
@@ -114,6 +114,10 @@ typedef struct bdb_entry_info {
 #endif
        Entry   *bei_e;
        Avlnode *bei_kids;
+#ifdef SLAP_ZONE_ALLOC
+       struct bdb_info *bei_bdb;
+       int bei_zseq;   
+#endif
        ldap_pvt_thread_mutex_t bei_kids_mutex;
        
        struct bdb_entry_info   *bei_lrunext;   /* for cache lru list */
@@ -135,6 +139,9 @@ typedef struct bdb_cache {
        ldap_pvt_thread_rdwr_t c_rwlock;
        ldap_pvt_thread_mutex_t lru_mutex;
        u_int32_t       c_locker;       /* used by lru cleaner */
+#ifdef SLAP_ZONE_ALLOC
+       void *c_zctx;
+#endif
 } Cache;
  
 #define CACHE_READ_LOCK                0

diff --git a/servers/slapd/back-bdb/cache.c b/servers/slapd/back-bdb/cache.c
index eaabdf020dbb604dc7217f2afe072b931da93fbe..1d9f4899bb93ea6c799694d53cf8040bdfa30598 100644 (file)
--- a/servers/slapd/back-bdb/cache.c
+++ b/servers/slapd/back-bdb/cache.c
@@ -230,6 +230,9 @@ bdb_entryinfo_add_internal(
 #ifdef BDB_HIER
        ei2->bei_rdn = ei->bei_rdn;
 #endif
+#ifdef SLAP_ZONE_ALLOC
+       ei2->bei_bdb = bdb;
+#endif
 
        /* Add to cache ID tree */
        if (avl_insert( &bdb->bi_cache.c_idtree, ei2, bdb_id_cmp, avl_dup_error )) {
@@ -393,6 +396,9 @@ hdb_cache_find_parent(
                ein->bei_nrdn = ei.bei_nrdn;
                ein->bei_rdn = ei.bei_rdn;
                ein->bei_ckids = ei.bei_ckids;
+#ifdef SLAP_ZONE_ALLOC
+               ein->bei_bdb = bdb;
+#endif
                ei.bei_ckids = 0;
                
                /* This node is not fully connected yet */
@@ -530,27 +536,32 @@ bdb_cache_lru_add(
                        elprev = elru->bei_lruprev;
 
                        /* Too many probes, not enough idle, give up */
-                       if (i > 10) break;
+                       if (i > 10)
+                               break;
 
                        /* If we can successfully writelock it, then
                         * the object is idle.
                         */
-                       if ( bdb_cache_entry_db_lock( bdb->bi_dbenv, bdb->bi_cache.c_locker, elru, 1, 1,
-                               lockp ) == 0 ) {
+                       if ( bdb_cache_entry_db_lock( bdb->bi_dbenv,
+                                       bdb->bi_cache.c_locker, elru, 1, 1, lockp ) == 0 ) {
                                int stop = 0;
 
                                /* If there's no entry, or this node is in
                                 * the process of linking into the cache,
                                 * or this node is being deleted, skip it.
                                 */
-                               if ( !elru->bei_e || (elru->bei_state & 
+                               if ( !elru->bei_e || (elru->bei_state &
                                        ( CACHE_ENTRY_NOT_LINKED | CACHE_ENTRY_DELETED ))) {
                                        bdb_cache_entry_db_unlock( bdb->bi_dbenv, lockp );
                                        continue;
                                }
                                LRU_DELETE( &bdb->bi_cache, elru );
                                elru->bei_e->e_private = NULL;
+#ifdef SLAP_ZONE_ALLOC
+                               bdb_entry_return( bdb, elru->bei_e, elru->bei_zseq );
+#else
                                bdb_entry_return( elru->bei_e );
+#endif
                                elru->bei_e = NULL;
                                ldap_pvt_thread_rdwr_wlock( &bdb->bi_cache.c_rwlock );
                                --bdb->bi_cache.c_cursize;
@@ -606,6 +617,9 @@ bdb_cache_find_id(
 
        ei.bei_id = id;
 
+#ifdef SLAP_ZONE_ALLOC
+       slap_zh_rlock(bdb->bi_cache.c_zctx);
+#endif
        /* If we weren't given any info, see if we have it already cached */
        if ( !*eip ) {
 again: ldap_pvt_thread_rdwr_rlock( &bdb->bi_cache.c_rwlock );
@@ -649,7 +663,11 @@ again:     ldap_pvt_thread_rdwr_rlock( &bdb->bi_cache.c_rwlock );
                                &ep->e_nname, eip );
                        if ( *eip ) islocked = 1;
                        if ( rc ) {
+#ifdef SLAP_ZONE_ALLOC
+                               bdb_entry_return( bdb, ep, (*eip)->bei_zseq );
+#else
                                bdb_entry_return( ep );
+#endif
                                ep = NULL;
                        }
                }
@@ -665,7 +683,15 @@ again:     ldap_pvt_thread_rdwr_rlock( &bdb->bi_cache.c_rwlock );
                        rc = DB_NOTFOUND;
                } else {
                        /* Make sure only one thread tries to load the entry */
-load1:         if ( !(*eip)->bei_e && !((*eip)->bei_state & CACHE_ENTRY_LOADING)) {
+load1:
+#ifdef SLAP_ZONE_ALLOC
+                       if ((*eip)->bei_e && !slap_zn_validate(
+                                       bdb->bi_cache.c_zctx, (*eip)->bei_e, (*eip)->bei_zseq)) {
+                               (*eip)->bei_e = NULL;
+                               (*eip)->bei_zseq = 0;
+                       }
+#endif
+                       if ( !(*eip)->bei_e && !((*eip)->bei_state & CACHE_ENTRY_LOADING)) {
                                load = 1;
                                (*eip)->bei_state |= CACHE_ENTRY_LOADING;
                        }
@@ -711,6 +737,9 @@ load1:              if ( !(*eip)->bei_e && !((*eip)->bei_state & CACHE_ENTRY_LOADING)) {
                                                bdb_fix_dn( ep, 0 );
 #endif
                                                (*eip)->bei_e = ep;
+#ifdef SLAP_ZONE_ALLOC
+                                               (*eip)->bei_zseq = *((ber_len_t *)ep - 2);
+#endif
                                                ep = NULL;
                                        }
                                        (*eip)->bei_state ^= CACHE_ENTRY_LOADING;
@@ -771,7 +800,11 @@ load1:             if ( !(*eip)->bei_e && !((*eip)->bei_state & CACHE_ENTRY_LOADING)) {
                bdb_cache_entryinfo_unlock( *eip );
        }
        if ( ep ) {
+#ifdef SLAP_ZONE_ALLOC
+               bdb_entry_return( bdb, ep, (*eip)->bei_zseq );
+#else
                bdb_entry_return( ep );
+#endif
        }
        if ( rc == 0 ) {
 
@@ -800,6 +833,12 @@ load1:             if ( !(*eip)->bei_e && !((*eip)->bei_state & CACHE_ENTRY_LOADING)) {
                }
        }
 
+#ifdef SLAP_ZONE_ALLOC
+       if (rc == 0 && (*eip)->bei_e) {
+               slap_zn_rlock(bdb->bi_cache.c_zctx, (*eip)->bei_e);
+       }
+       slap_zh_runlock(bdb->bi_cache.c_zctx);
+#endif
        return rc;
 }
 
@@ -865,7 +904,11 @@ bdb_cache_add(
        /* bdb_csn_commit can cause this when adding the database root entry */
        if ( new->bei_e ) {
                new->bei_e->e_private = NULL;
+#ifdef SLAP_ZONE_ALLOC
+               bdb_entry_return( bdb, new->bei_e, new->bei_zseq );
+#else
                bdb_entry_return( new->bei_e );
+#endif
        }
        new->bei_e = e;
        e->e_private = new;
@@ -1043,7 +1086,11 @@ bdb_cache_delete_cleanup(
 {
        if ( ei->bei_e ) {
                ei->bei_e->e_private = NULL;
+#ifdef SLAP_ZONE_ALLOC
+               bdb_entry_return( ei->bei_bdb, ei->bei_e, ei->bei_zseq );
+#else
                bdb_entry_return( ei->bei_e );
+#endif
                ei->bei_e = NULL;
        }
 
@@ -1123,7 +1170,11 @@ bdb_entryinfo_release( void *data )
        }
        if ( ei->bei_e ) {
                ei->bei_e->e_private = NULL;
+#ifdef SLAP_ZONE_ALLOC
+               bdb_entry_return( ei->bei_bdb, ei->bei_e, ei->bei_zseq );
+#else
                bdb_entry_return( ei->bei_e );
+#endif
        }
        bdb_cache_entryinfo_destroy( ei );
 }
@@ -1310,7 +1361,11 @@ bdb_cache_delete_entry(
                if ( ei->bei_e && !(ei->bei_state & CACHE_ENTRY_NOT_LINKED )) {
                        LRU_DELETE( &bdb->bi_cache, ei );
                        ei->bei_e->e_private = NULL;
+#ifdef SLAP_ZONE_ALLOC
+                       bdb_entry_return( bdb, ei->bei_e, ei->bei_zseq );
+#else
                        bdb_entry_return( ei->bei_e );
+#endif
                        ei->bei_e = NULL;
                        --bdb->bi_cache.c_cursize;
                }

diff --git a/servers/slapd/back-bdb/id2entry.c b/servers/slapd/back-bdb/id2entry.c
index 1ef56e71d330422e24e013f95f3a50f2a606046c..ba63ccea38630dabef6350e3ccef6162899038f2 100644 (file)
--- a/servers/slapd/back-bdb/id2entry.c
+++ b/servers/slapd/back-bdb/id2entry.c
@@ -121,7 +121,11 @@ int bdb_id2entry(
 
        DBT2bv( &data, &bv );
 
-       rc = entry_decode( &bv, e );
+#ifdef SLAP_ZONE_ALLOC
+       rc = entry_decode(&bv, e, bdb->bi_cache.c_zctx);
+#else
+       rc = entry_decode(&bv, e);
+#endif
 
        if( rc == 0 ) {
                (*e)->e_id = id;
@@ -129,8 +133,13 @@ int bdb_id2entry(
                /* only free on error. On success, the entry was
                 * decoded in place.
                 */
-               ch_free( data.data );
+#ifndef SLAP_ZONE_ALLOC
+               ch_free(data.data);
+#endif
        }
+#ifdef SLAP_ZONE_ALLOC
+       ch_free(data.data);
+#endif
 
        return rc;
 }
@@ -157,9 +166,23 @@ int bdb_id2entry_delete(
        return rc;
 }
 
+#ifdef SLAP_ZONE_ALLOC
 int bdb_entry_return(
-       Entry *e )
+       struct bdb_info *bdb,
+       Entry *e,
+       int zseq
+)
+#else
+int bdb_entry_return(
+       Entry *e
+)
+#endif
 {
+#ifdef SLAP_ZONE_ALLOC
+       if (!slap_zn_validate(bdb->bi_cache.c_zctx, e, zseq)) {
+               return 0;
+       }
+#endif
        /* Our entries are allocated in two blocks; the data comes from
         * the db itself and the Entry structure and associated pointers
         * are allocated in entry_decode. The db data pointer is saved
@@ -168,10 +191,11 @@ int bdb_entry_return(
         * is when an entry has been modified, in which case we also need
         * to free e_attrs.
         */
+
 #ifdef LDAP_COMP_MATCH
        comp_tree_free( e->e_attrs );
 #endif
-       if( !e->e_bv.bv_val ) { /* A regular entry, from do_add */
+       if( !e->e_bv.bv_val ) { /* Entry added by do_add */
                entry_free( e );
                return 0;
        }
@@ -187,6 +211,7 @@ int bdb_entry_return(
                e->e_name.bv_val = NULL;
                e->e_nname.bv_val = NULL;
        }
+#ifndef SLAP_ZONE_ALLOC
 #ifndef BDB_HIER
        /* In tool mode the e_bv buffer is realloc'd, leave it alone */
        if( !(slapMode & SLAP_TOOL_MODE) ) {
@@ -194,18 +219,24 @@ int bdb_entry_return(
        }
 #else
        free( e->e_bv.bv_val );
-#endif
+#endif /* BDB_HIER */
+#endif /* !SLAP_ZONE_ALLOC */
+
+#ifdef SLAP_ZONE_ALLOC
+       slap_zn_free( e, bdb->bi_cache.c_zctx );
+#else
        free( e );
+#endif
 
        return 0;
 }
 
 int bdb_entry_release(
-       Operation *o,
+       Operation *op,
        Entry *e,
        int rw )
 {
-       struct bdb_info *bdb = (struct bdb_info *) o->o_bd->be_private;
+       struct bdb_info *bdb = (struct bdb_info *) op->o_bd->be_private;
        struct bdb_op_info *boi = NULL;
  
        /* slapMode : SLAP_SERVER_MODE, SLAP_TOOL_MODE,
@@ -214,11 +245,15 @@ int bdb_entry_release(
        if ( slapMode == SLAP_SERVER_MODE ) {
                /* If not in our cache, just free it */
                if ( !e->e_private ) {
+#ifdef SLAP_ZONE_ALLOC
+                       return bdb_entry_return( bdb, e, -1 );
+#else
                        return bdb_entry_return( e );
+#endif
                }
                /* free entry and reader or writer lock */
-               if ( o ) {
-                       boi = (struct bdb_op_info *)o->o_private;
+               if ( op ) {
+                       boi = (struct bdb_op_info *)op->o_private;
                }
                /* lock is freed with txn */
                if ( !boi || boi->boi_txn ) {
@@ -231,20 +266,32 @@ int bdb_entry_release(
                                        bdb_cache_return_entry_rw( bdb->bi_dbenv, &bdb->bi_cache,
                                                e, rw, &bli->bli_lock );
                                        prev->bli_next = bli->bli_next;
-                                       o->o_tmpfree( bli, o->o_tmpmemctx );
+                                       op->o_tmpfree( bli, op->o_tmpmemctx );
                                        break;
                                }
                        }
                        if ( !boi->boi_locks ) {
-                               o->o_tmpfree( boi, o->o_tmpmemctx );
-                               o->o_private = NULL;
+                               op->o_tmpfree( boi, op->o_tmpmemctx );
+                               op->o_private = NULL;
                        }
                }
        } else {
+#ifdef SLAP_ZONE_ALLOC
+               int zseq = -1;
+               if (e->e_private != NULL) {
+                       BEI(e)->bei_e = NULL;
+                       zseq = BEI(e)->bei_zseq;
+               }
+#else
                if (e->e_private != NULL)
                        BEI(e)->bei_e = NULL;
+#endif
                e->e_private = NULL;
+#ifdef SLAP_ZONE_ALLOC
+               bdb_entry_return ( bdb, e, zseq );
+#else
                bdb_entry_return ( e );
+#endif
        }
  
        return 0;

diff --git a/servers/slapd/back-bdb/init.c b/servers/slapd/back-bdb/init.c
index c8a351ca1d9d9792e849af3b37cc2e1baf427d43..6733592bf5d805e4eaf964b0f01493c13127821f 100644 (file)
--- a/servers/slapd/back-bdb/init.c
+++ b/servers/slapd/back-bdb/init.c
@@ -192,6 +192,16 @@ bdb_db_open( BackendDB *be )
        /* One long-lived TXN per thread, two TXNs per write op */
        bdb->bi_dbenv->set_tx_max( bdb->bi_dbenv, connection_pool_max * 3 );
 
+#ifdef SLAP_ZONE_ALLOC
+       if ( bdb->bi_cache.c_maxsize ) {
+               bdb->bi_cache.c_zctx = slap_zn_mem_create(
+                                                               SLAP_ZONE_INITSIZE,
+                                                               SLAP_ZONE_MAXSIZE,
+                                                               SLAP_ZONE_DELTA,
+                                                               SLAP_ZONE_SIZE);
+       }
+#endif
+
        if ( bdb->bi_idl_cache_max_size ) {
                bdb->bi_idl_tree = NULL;
                ldap_pvt_thread_rdwr_init( &bdb->bi_idl_tree_rwlock );

diff --git a/servers/slapd/back-bdb/proto-bdb.h b/servers/slapd/back-bdb/proto-bdb.h
index 7cc491647899d1e8d455001cb0b0e6f5dbeaca03..8480db421ed83b234f628f65ff1bda6e5323ccb4 100644 (file)
--- a/servers/slapd/back-bdb/proto-bdb.h
+++ b/servers/slapd/back-bdb/proto-bdb.h
@@ -189,11 +189,14 @@ int bdb_id2entry_delete(
        DB_TXN *tid,
        Entry *e);
 
+#ifdef SLAP_ZONE_ALLOC
+#else
 int bdb_id2entry(
        BackendDB *be,
        DB_TXN *tid,
        ID id,
        Entry **e);
+#endif
 
 #define bdb_entry_free                         BDB_SYMBOL(entry_free)
 #define bdb_entry_return                       BDB_SYMBOL(entry_return)
@@ -201,7 +204,11 @@ int bdb_id2entry(
 #define bdb_entry_get                          BDB_SYMBOL(entry_get)
 
 void bdb_entry_free ( Entry *e );
+#ifdef SLAP_ZONE_ALLOC
+int bdb_entry_return( struct bdb_info *bdb, Entry *e, int seqno );
+#else
 int bdb_entry_return( Entry *e );
+#endif
 BI_entry_release_rw bdb_entry_release;
 BI_entry_get_rw bdb_entry_get;
 
@@ -427,7 +434,7 @@ void bdb_unlocked_cache_return_entry_rw( Cache *cache, Entry *e, int rw );
 #define bdb_cache_add                          BDB_SYMBOL(cache_add)
 #define bdb_cache_children                     BDB_SYMBOL(cache_children)
 #define bdb_cache_delete                       BDB_SYMBOL(cache_delete)
-#define bdb_cache_delete_cleanup               BDB_SYMBOL(cache_delete_cleanup)
+#define bdb_cache_delete_cleanup       BDB_SYMBOL(cache_delete_cleanup)
 #define bdb_cache_find_id                      BDB_SYMBOL(cache_find_id)
 #define bdb_cache_find_info                    BDB_SYMBOL(cache_find_info)
 #define bdb_cache_find_ndn                     BDB_SYMBOL(cache_find_ndn)

diff --git a/servers/slapd/back-bdb/search.c b/servers/slapd/back-bdb/search.c
index 0c99ffddbc4aea4d3013b5a6b50e599ffc981004..e47ce1f86b60ced155cf875f6c7ebcc9bfcc7ae6 100644 (file)
--- a/servers/slapd/back-bdb/search.c
+++ b/servers/slapd/back-bdb/search.c
@@ -842,6 +842,9 @@ dn2entry_retry:
                                ? get_entry_referrals( op, matched )
                                : NULL;
 
+#ifdef SLAP_ZONE_ALLOC
+                       slap_zn_runlock(bdb->bi_cache.c_zctx, matched);
+#endif
                        bdb_cache_return_entry_r (bdb->bi_dbenv, &bdb->bi_cache,
                                matched, &lock);
                        matched = NULL;
@@ -853,6 +856,9 @@ dn2entry_retry:
                        }
 
                } else {
+#ifdef SLAP_ZONE_ALLOC
+                       slap_zn_runlock(bdb->bi_cache.c_zctx, matched);
+#endif
                        rs->sr_ref = referral_rewrite( default_referral,
                                NULL, &sop->o_req_dn, sop->oq_search.rs_scope );
                }
@@ -882,6 +888,9 @@ dn2entry_retry:
                ber_dupbv( &matched_dn, &e->e_name );
                erefs = get_entry_referrals( op, e );
 
+#ifdef SLAP_ZONE_ALLOC
+               slap_zn_runlock(bdb->bi_cache.c_zctx, e);
+#endif
                bdb_cache_return_entry_r( bdb->bi_dbenv, &bdb->bi_cache, e, &lock );
                e = NULL;
 
@@ -913,6 +922,9 @@ dn2entry_retry:
                ( test_filter( op, e, get_assertion( op )) != LDAP_COMPARE_TRUE ))
        {
                rs->sr_err = LDAP_ASSERTION_FAILED;
+#ifdef SLAP_ZONE_ALLOC
+               slap_zn_runlock(bdb->bi_cache.c_zctx, e);
+#endif
                send_ldap_result( sop, rs );
                return 1;
        }
@@ -930,6 +942,9 @@ dn2entry_retry:
        base.e_nname = realbase;
        base.e_id = e->e_id;
 
+#ifdef SLAP_ZONE_ALLOC
+       slap_zn_runlock(bdb->bi_cache.c_zctx, e);
+#endif
        if ( e != &e_root ) {
                bdb_cache_return_entry_r(bdb->bi_dbenv, &bdb->bi_cache, e, &lock);
        }
@@ -1230,7 +1245,7 @@ loop_begin:
 #ifdef BDB_PSEARCH
                if (!IS_PSEARCH) {
 #endif
-id2entry_retry:
+fetch_entry_retry:
                        /* get the entry with reader lock */
                        ei = NULL;
                        rs->sr_err = bdb_cache_find_id( op, ltid,
@@ -1244,7 +1259,7 @@ id2entry_retry:
                        } else if ( rs->sr_err == DB_LOCK_DEADLOCK
                                || rs->sr_err == DB_LOCK_NOTGRANTED )
                        {
-                               goto id2entry_retry;    
+                               goto fetch_entry_retry;
                        }
 
                        if ( ei && rs->sr_err == LDAP_SUCCESS ) {
@@ -1471,6 +1486,9 @@ id2entry_retry:
                         ) {
 #ifdef BDB_PSEARCH
                                if (!IS_PSEARCH) {
+#endif
+#ifdef SLAP_ZONE_ALLOC
+                                       slap_zn_runlock(bdb->bi_cache.c_zctx, e);
 #endif
                                        bdb_cache_return_entry_r( bdb->bi_dbenv,
                                                &bdb->bi_cache, e, &lock );
@@ -1670,6 +1688,9 @@ post_search_no_entry:
                                case -1:        /* connection closed */
 #ifdef BDB_PSEARCH
                                        if (!IS_PSEARCH) {
+#endif
+#ifdef SLAP_ZONE_ALLOC
+                                               slap_zn_runlock(bdb->bi_cache.c_zctx, e);
 #endif
                                                bdb_cache_return_entry_r(bdb->bi_dbenv,
                                                        &bdb->bi_cache, e, &lock);
@@ -1697,6 +1718,9 @@ loop_continue:
                        if (!IS_PSEARCH) {
                                if (!(IS_POST_SEARCH &&
                                                 entry_sync_state == LDAP_SYNC_DELETE)) {
+#ifdef SLAP_ZONE_ALLOC
+                                       slap_zn_runlock(bdb->bi_cache.c_zctx, e);
+#endif
                                        bdb_cache_return_entry_r( bdb->bi_dbenv,
                                                &bdb->bi_cache, e , &lock );
                                        if ( sop->o_nocaching ) {
@@ -1705,6 +1729,9 @@ loop_continue:
                                }
                        }
 #else
+#ifdef SLAP_ZONE_ALLOC
+                       slap_zn_runlock(bdb->bi_cache.c_zctx, e);
+#endif
                        bdb_cache_return_entry_r( bdb->bi_dbenv,
                                &bdb->bi_cache, e , &lock );
 #endif
@@ -1867,6 +1894,9 @@ done:
 
        if( !IS_PSEARCH && e != NULL ) {
                /* free reader lock */
+#ifdef SLAP_ZONE_ALLOC
+               slap_zn_runlock(bdb->bi_cache.c_zctx, e);
+#endif
                bdb_cache_return_entry_r( bdb->bi_dbenv, &bdb->bi_cache, e, &lock );
        }
        ber_bvfree( search_context_csn );

diff --git a/servers/slapd/back-bdb/tools.c b/servers/slapd/back-bdb/tools.c
index d76aef4dc1365de56ee27f558b9e97b972c1c993..033b0bd8bd6a350aee1e6ada2fb601ad00656b4f 100644 (file)
--- a/servers/slapd/back-bdb/tools.c
+++ b/servers/slapd/back-bdb/tools.c
@@ -176,7 +176,12 @@ Entry* bdb_tool_entry_get( BackendDB *be, ID id )
 #ifndef BDB_HIER
        DBT2bv( &data, &bv );
 
+#ifdef SLAP_ZONE_ALLOC
+       /* FIXME: will add ctx later */
+       rc = entry_decode( &bv, &e, NULL );
+#else
        rc = entry_decode( &bv, &e );
+#endif
 
        if( rc == LDAP_SUCCESS ) {
                e->e_id = id;