X-Git-Url: https://git.sur5r.net/?a=blobdiff_plain;f=servers%2Fslapd%2Foverlays%2Fpcache.c;h=450706c12751e13454798016afecc6057b54fb95;hb=2c3aaca62f060cd276cdeb6ad7da617962234aa3;hp=fdf341b0cd2de88948cd552bd0457e01adce1daa;hpb=8be1b48263f98f85eebaebbb269311a9b4bea5ac;p=openldap diff --git a/servers/slapd/overlays/pcache.c b/servers/slapd/overlays/pcache.c index fdf341b0cd..450706c127 100644 --- a/servers/slapd/overlays/pcache.c +++ b/servers/slapd/overlays/pcache.c @@ -1,7 +1,7 @@ /* $OpenLDAP$ */ /* This work is part of OpenLDAP Software . * - * Copyright 2003-2007 The OpenLDAP Foundation. + * Copyright 2003-2008 The OpenLDAP Foundation. * Portions Copyright 2003 IBM Corporation. * Portions Copyright 2003 Symas Corporation. * All rights reserved. @@ -72,12 +72,14 @@ typedef struct cached_query_s { Qbase *qbase; int scope; struct berval q_uuid; /* query identifier */ + int q_sizelimit; struct query_template_s *qtemp; /* template of the query */ - time_t expiry_time; /* time till the query is considered valid */ + time_t expiry_time; /* time till the query is considered valid */ struct cached_query_s *next; /* next query in the template */ struct cached_query_s *prev; /* previous query in the template */ - struct cached_query_s *lru_up; /* previous query in the LRU list */ - struct cached_query_s *lru_down; /* next query in the LRU list */ + struct cached_query_s *lru_up; /* previous query in the LRU list */ + struct cached_query_s *lru_down; /* next query in the LRU list */ + ldap_pvt_thread_rdwr_t rwlock; } CachedQuery; /* @@ -127,17 +129,36 @@ typedef struct query_template_s { int no_of_queries; /* Total number of queries in the template */ time_t ttl; /* TTL for the queries of this template */ time_t negttl; /* TTL for negative results */ + time_t limitttl; /* TTL for sizelimit exceeding results */ struct attr_set t_attrs; /* filter attrs + attr_set */ } QueryTemplate; +typedef enum { + PC_IGNORE = 0, + PC_POSITIVE, + PC_NEGATIVE, + PC_SIZELIMIT +} pc_caching_reason_t; + +static const char *pc_caching_reason_str[] = { + "IGNORE", + "POSITIVE", + "NEGATIVE", + "SIZELIMIT", + + NULL +}; + struct query_manager_s; /* prototypes for functions for 1) query containment * 2) query addition, 3) cache replacement */ -typedef CachedQuery * (QCfunc)(Operation *op, struct query_manager_s*, Query*, QueryTemplate*); -typedef CachedQuery * (AddQueryfunc)(Operation *op, struct query_manager_s*, Query*, QueryTemplate*, int positive); -typedef void (CRfunc)(struct query_manager_s*, struct berval * ); +typedef CachedQuery *(QCfunc)(Operation *op, struct query_manager_s*, + Query*, QueryTemplate*); +typedef CachedQuery *(AddQueryfunc)(Operation *op, struct query_manager_s*, + Query*, QueryTemplate*, pc_caching_reason_t, int wlock); +typedef void (CRfunc)(struct query_manager_s*, struct berval*); /* LDAP query cache */ typedef struct query_manager_s { @@ -164,15 +185,16 @@ typedef struct cache_manager_s { int numattrsets; /* number of attribute sets */ int cur_entries; /* current number of entries cached */ int max_entries; /* max number of entries cached */ - int num_entries_limit; /* max # of entries in a cacheable query */ + int num_entries_limit; /* max # of entries in a cacheable query */ char response_cb; /* install the response callback * at the tail of the callback list */ #define PCACHE_RESPONSE_CB_HEAD 0 #define PCACHE_RESPONSE_CB_TAIL 1 + char defer_db_open; /* defer open for online add */ time_t cc_period; /* interval between successive consistency checks (sec) */ - int cc_paused; + int cc_paused; void *cc_arg; ldap_pvt_thread_mutex_t cache_mutex; @@ -225,7 +247,8 @@ add_query( query_manager* qm, Query* query, QueryTemplate *templ, - int positive); + pc_caching_reason_t why, + int wlock); static int remove_query_data( @@ -452,7 +475,7 @@ url2query( goto error; } - cq = add_query( op, qm, &query, qt, 1 ); + cq = add_query( op, qm, &query, qt, PC_POSITIVE, 0 ); if ( cq != NULL ) { cq->expiry_time = expiry_time; cq->q_uuid = uuid; @@ -1093,6 +1116,10 @@ query_containment(Operation *op, query_manager *qm, qc = find_filter( op, qbptr->scopes[tscope], query->filter, first ); if ( qc ) { + if ( qc->q_sizelimit ) { + ldap_pvt_thread_rdwr_runlock(&templa->t_rwlock); + return NULL; + } ldap_pvt_thread_mutex_lock(&qm->lru_mutex); if (qm->lru_top != qc) { remove_query(qm, qc); @@ -1128,35 +1155,57 @@ free_query (CachedQuery* qc) } -/* Add query to query cache */ +/* Add query to query cache, the returned Query is locked for writing */ static CachedQuery * add_query( Operation *op, query_manager* qm, Query* query, QueryTemplate *templ, - int positive) + pc_caching_reason_t why, + int wlock) { CachedQuery* new_cached_query = (CachedQuery*) ch_malloc(sizeof(CachedQuery)); Qbase *qbase, qb; Filter *first; int rc; + time_t ttl = 0;; new_cached_query->qtemp = templ; BER_BVZERO( &new_cached_query->q_uuid ); - if ( positive ) { - new_cached_query->expiry_time = slap_get_time() + templ->ttl; - } else { - new_cached_query->expiry_time = slap_get_time() + templ->negttl; + new_cached_query->q_sizelimit = 0; + + switch ( why ) { + case PC_POSITIVE: + ttl = templ->ttl; + break; + + case PC_NEGATIVE: + ttl = templ->negttl; + break; + + case PC_SIZELIMIT: + ttl = templ->limitttl; + break; + + default: + assert( 0 ); + break; } + new_cached_query->expiry_time = slap_get_time() + ttl; new_cached_query->lru_up = NULL; new_cached_query->lru_down = NULL; - Debug( pcache_debug, "Added query expires at %ld\n", - (long) new_cached_query->expiry_time, 0, 0 ); + Debug( pcache_debug, "Added query expires at %ld (%s)\n", + (long) new_cached_query->expiry_time, + pc_caching_reason_str[ why ], 0 ); new_cached_query->scope = query->scope; new_cached_query->filter = query->filter; new_cached_query->first = first = filter_first( query->filter ); + + ldap_pvt_thread_rdwr_init(&new_cached_query->rwlock); + if (wlock) + ldap_pvt_thread_rdwr_wlock(&new_cached_query->rwlock); qb.base = query->base; @@ -1286,7 +1335,7 @@ cache_replacement(query_manager* qm, struct berval *result) ldap_pvt_thread_mutex_unlock(&qm->lru_mutex); *result = bottom->q_uuid; - bottom->q_uuid.bv_val = NULL; + BER_BVZERO( &bottom->q_uuid ); Debug( pcache_debug, "Lock CR index = %p\n", (void *) temp, 0, 0 ); ldap_pvt_thread_rdwr_wlock(&temp->t_rwlock); @@ -1319,8 +1368,7 @@ remove_func ( attr = attr_find( rs->sr_entry->e_attrs, ad_queryId ); if ( attr == NULL ) return 0; - for ( count = 0; !BER_BVISNULL( &attr->a_vals[count] ); count++ ) - ; + count = attr->a_numvals; assert( count > 0 ); qi = op->o_tmpalloc( sizeof( struct query_info ), op->o_tmpmemctx ); qi->next = op->o_callback->sc_private; @@ -1339,11 +1387,7 @@ remove_query_data( { struct query_info *qi, *qnext; char filter_str[ LDAP_LUTIL_UUIDSTR_BUFSIZE + STRLENOF( "(queryId=)" ) ]; -#ifdef LDAP_COMP_MATCH - AttributeAssertion ava = { NULL, BER_BVNULL, NULL }; -#else - AttributeAssertion ava = { NULL, BER_BVNULL }; -#endif + AttributeAssertion ava = ATTRIBUTEASSERTION_INIT; Filter filter = {LDAP_FILTER_EQUALITY}; SlapReply sreply = {REP_RESULT}; slap_callback cb = { NULL, remove_func, NULL, NULL }; @@ -1406,6 +1450,7 @@ remove_query_data( mod.sml_type = ad_queryId->ad_cname; mod.sml_values = vals; mod.sml_nvalues = NULL; + mod.sml_numvals = 1; mod.sml_next = NULL; Debug( pcache_debug, "REMOVING TEMP ATTR : TEMPLATE=%s\n", @@ -1438,67 +1483,91 @@ filter2template( int* filter_got_oc ) { AttributeDescription *ad; + int len, ret; switch ( f->f_choice ) { case LDAP_FILTER_EQUALITY: ad = f->f_av_desc; - sprintf( fstr->bv_val+fstr->bv_len, "(%s=)", ad->ad_cname.bv_val ); - fstr->bv_len += ad->ad_cname.bv_len + ( sizeof("(=)") - 1 ); + len = STRLENOF( "(=)" ) + ad->ad_cname.bv_len; + ret = snprintf( fstr->bv_val+fstr->bv_len, len + 1, "(%s=)", ad->ad_cname.bv_val ); + assert( ret == len ); + fstr->bv_len += len; break; case LDAP_FILTER_GE: ad = f->f_av_desc; - sprintf( fstr->bv_val+fstr->bv_len, "(%s>=)", ad->ad_cname.bv_val); - fstr->bv_len += ad->ad_cname.bv_len + ( sizeof("(>=)") - 1 ); + len = STRLENOF( "(>=)" ) + ad->ad_cname.bv_len; + ret = snprintf( fstr->bv_val+fstr->bv_len, len + 1, "(%s>=)", ad->ad_cname.bv_val); + assert( ret == len ); + fstr->bv_len += len; break; case LDAP_FILTER_LE: ad = f->f_av_desc; - sprintf( fstr->bv_val+fstr->bv_len, "(%s<=)", ad->ad_cname.bv_val); - fstr->bv_len += ad->ad_cname.bv_len + ( sizeof("(<=)") - 1 ); + len = STRLENOF( "(<=)" ) + ad->ad_cname.bv_len; + ret = snprintf( fstr->bv_val+fstr->bv_len, len + 1, "(%s<=)", ad->ad_cname.bv_val); + assert( ret == len ); + fstr->bv_len += len; break; case LDAP_FILTER_APPROX: ad = f->f_av_desc; - sprintf( fstr->bv_val+fstr->bv_len, "(%s~=)", ad->ad_cname.bv_val); - fstr->bv_len += ad->ad_cname.bv_len + ( sizeof("(~=)") - 1 ); + len = STRLENOF( "(~=)" ) + ad->ad_cname.bv_len; + ret = snprintf( fstr->bv_val+fstr->bv_len, len + 1, "(%s~=)", ad->ad_cname.bv_val); + assert( ret == len ); + fstr->bv_len += len; break; case LDAP_FILTER_SUBSTRINGS: ad = f->f_sub_desc; - sprintf( fstr->bv_val+fstr->bv_len, "(%s=)", ad->ad_cname.bv_val ); - fstr->bv_len += ad->ad_cname.bv_len + ( sizeof("(=)") - 1 ); + len = STRLENOF( "(=)" ) + ad->ad_cname.bv_len; + ret = snprintf( fstr->bv_val+fstr->bv_len, len + 1, "(%s=)", ad->ad_cname.bv_val ); + assert( ret == len ); + fstr->bv_len += len; break; case LDAP_FILTER_PRESENT: ad = f->f_desc; - sprintf( fstr->bv_val+fstr->bv_len, "(%s=*)", ad->ad_cname.bv_val ); - fstr->bv_len += ad->ad_cname.bv_len + ( sizeof("(=*)") - 1 ); + len = STRLENOF( "(=*)" ) + ad->ad_cname.bv_len; + ret = snprintf( fstr->bv_val+fstr->bv_len, len + 1, "(%s=*)", ad->ad_cname.bv_val ); + assert( ret == len ); + fstr->bv_len += len; break; case LDAP_FILTER_AND: case LDAP_FILTER_OR: case LDAP_FILTER_NOT: { int rc = 0; - sprintf( fstr->bv_val+fstr->bv_len, "(%c", - f->f_choice == LDAP_FILTER_AND ? '&' : - f->f_choice == LDAP_FILTER_OR ? '|' : '!' ); - fstr->bv_len += sizeof("(%") - 1; + fstr->bv_val[fstr->bv_len++] = '('; + switch ( f->f_choice ) { + case LDAP_FILTER_AND: + fstr->bv_val[fstr->bv_len] = '&'; + break; + case LDAP_FILTER_OR: + fstr->bv_val[fstr->bv_len] = '|'; + break; + case LDAP_FILTER_NOT: + fstr->bv_val[fstr->bv_len] = '!'; + break; + } + fstr->bv_len++; for ( f = f->f_list; f != NULL; f = f->f_next ) { rc = filter2template( op, f, fstr, filter_attrs, filter_cnt, filter_got_oc ); if ( rc ) break; } - sprintf( fstr->bv_val+fstr->bv_len, ")" ); - fstr->bv_len += sizeof(")") - 1; + fstr->bv_val[fstr->bv_len++] = ')'; + fstr->bv_val[fstr->bv_len] = '\0'; return rc; } default: - strcpy( fstr->bv_val, "(?=?)" ); - fstr->bv_len += sizeof("(?=?)")-1; + /* a filter should at least have room for "()", + * an "=" and for a 1-char attr */ + strcpy( fstr->bv_val, "(?=)" ); + fstr->bv_len += STRLENOF("(?=)"); return -1; } @@ -1527,6 +1596,9 @@ struct search_info { int max; int over; int count; + int slimit; + int slimit_exceeded; + pc_caching_reason_t caching_reason; Entry *head, *tail; }; @@ -1613,7 +1685,7 @@ int pcache_remove_entries_from_cache( Operation *op, cache_manager *cm, - BerVarray UUIDs ) + BerVarray entryUUIDs ) { Connection conn = { 0 }; OperationBuffer opbuf; @@ -1622,11 +1694,7 @@ pcache_remove_entries_from_cache( SlapReply rs = { REP_RESULT }; Filter f = { 0 }; char filtbuf[ LDAP_LUTIL_UUIDSTR_BUFSIZE + STRLENOF( "(entryUUID=)" ) ]; -#ifdef LDAP_COMP_MATCH - AttributeAssertion ava = { NULL, BER_BVNULL, NULL }; -#else - AttributeAssertion ava = { NULL, BER_BVNULL }; -#endif + AttributeAssertion ava = ATTRIBUTEASSERTION_INIT; AttributeName attrs[ 2 ] = { 0 }; int s, rc; @@ -1667,13 +1735,13 @@ pcache_remove_entries_from_cache( sc.sc_response = fetch_queryId_cb; op->o_callback = ≻ - for ( s = 0; !BER_BVISNULL( &UUIDs[ s ] ); s++ ) { + for ( s = 0; !BER_BVISNULL( &entryUUIDs[ s ] ); s++ ) { BerVarray vals = NULL; op->ors_filterstr.bv_len = snprintf( filtbuf, sizeof( filtbuf ), - "(entryUUID=%s)", UUIDs[ s ].bv_val ); + "(entryUUID=%s)", entryUUIDs[ s ].bv_val ); op->ors_filterstr.bv_val = filtbuf; - ava.aa_value = UUIDs[ s ]; + ava.aa_value = entryUUIDs[ s ]; rc = op->o_bd->be_search( op, &rs ); if ( rc != LDAP_SUCCESS ) { @@ -1703,14 +1771,36 @@ pcache_remove_entries_from_cache( } /* - * Call that allows to remove a set of queries from the cache + * Call that allows to remove a query from the cache. + */ +int +pcache_remove_query_from_cache( + Operation *op, + cache_manager *cm, + struct berval *queryid ) +{ + Operation op2 = *op; + SlapReply rs2 = { 0 }; + + op2.o_bd = &cm->db; + + /* remove the selected query */ + remove_query_and_data( &op2, &rs2, cm, queryid ); + + return LDAP_SUCCESS; +} + +/* + * Call that allows to remove a set of queries related to an entry + * from the cache; if queryid is not null, the entry must belong to + * the query indicated by queryid. */ int pcache_remove_entry_queries_from_cache( Operation *op, cache_manager *cm, struct berval *ndn, - struct berval *uuid ) + struct berval *queryid ) { Connection conn = { 0 }; OperationBuffer opbuf; @@ -1719,11 +1809,7 @@ pcache_remove_entry_queries_from_cache( SlapReply rs = { REP_RESULT }; Filter f = { 0 }; char filter_str[ LDAP_LUTIL_UUIDSTR_BUFSIZE + STRLENOF( "(queryId=)" ) ]; -#ifdef LDAP_COMP_MATCH - AttributeAssertion ava = { NULL, BER_BVNULL, NULL }; -#else - AttributeAssertion ava = { NULL, BER_BVNULL }; -#endif + AttributeAssertion ava = ATTRIBUTEASSERTION_INIT; AttributeName attrs[ 2 ] = { 0 }; int rc; @@ -1743,7 +1829,7 @@ pcache_remove_entry_queries_from_cache( memset( &op->oq_search, 0, sizeof( op->oq_search ) ); op->ors_scope = LDAP_SCOPE_BASE; op->ors_deref = LDAP_DEREF_NEVER; - if ( uuid == NULL || BER_BVISNULL( uuid ) ) { + if ( queryid == NULL || BER_BVISNULL( queryid ) ) { BER_BVSTR( &op->ors_filterstr, "(objectClass=*)" ); f.f_choice = LDAP_FILTER_PRESENT; f.f_desc = slap_schema.si_ad_objectClass; @@ -1751,11 +1837,11 @@ pcache_remove_entry_queries_from_cache( } else { op->ors_filterstr.bv_len = snprintf( filter_str, sizeof( filter_str ), "(%s=%s)", - ad_queryId->ad_cname.bv_val, uuid->bv_val ); + ad_queryId->ad_cname.bv_val, queryid->bv_val ); f.f_choice = LDAP_FILTER_EQUALITY; f.f_ava = &ava; f.f_av_desc = ad_queryId; - f.f_av_value = *uuid; + f.f_av_value = *queryid; } op->ors_filter = &f; op->ors_slimit = 1; @@ -1815,14 +1901,19 @@ cache_entries( Entry *e; struct berval crp_uuid; char uuidbuf[ LDAP_LUTIL_UUIDSTR_BUFSIZE ]; - Operation op_tmp = *op; + Operation *op_tmp; + Connection conn = {0}; + OperationBuffer opbuf; + void *thrctx = ldap_pvt_thread_pool_context(); query_uuid->bv_len = lutil_uuidstr(uuidbuf, sizeof(uuidbuf)); ber_str2bv(uuidbuf, query_uuid->bv_len, 1, query_uuid); - op_tmp.o_bd = &cm->db; - op_tmp.o_dn = cm->db.be_rootdn; - op_tmp.o_ndn = cm->db.be_rootndn; + connection_fake_init2( &conn, &opbuf, thrctx, 0 ); + op_tmp = &opbuf.ob_op; + op_tmp->o_bd = &cm->db; + op_tmp->o_dn = cm->db.be_rootdn; + op_tmp->o_ndn = cm->db.be_rootndn; Debug( pcache_debug, "UUID for query being added = %s\n", uuidbuf, 0, 0 ); @@ -1832,10 +1923,10 @@ cache_entries( e->e_private = NULL; while ( cm->cur_entries > (cm->max_entries) ) { BER_BVZERO( &crp_uuid ); - remove_query_and_data( &op_tmp, rs, cm, &crp_uuid ); + remove_query_and_data( op_tmp, rs, cm, &crp_uuid ); } - return_val = merge_entry(&op_tmp, e, query_uuid); + return_val = merge_entry(op_tmp, e, query_uuid); ldap_pvt_thread_mutex_lock(&cm->cache_mutex); cm->cur_entries += return_val; Debug( pcache_debug, @@ -1852,32 +1943,18 @@ static int pcache_op_cleanup( Operation *op, SlapReply *rs ) { slap_callback *cb = op->o_callback; struct search_info *si = cb->sc_private; - if ( si->save_attrs != NULL ) { - rs->sr_attrs = si->save_attrs; - op->ors_attrs = si->save_attrs; - } - op->o_callback = op->o_callback->sc_next; - op->o_tmpfree( cb, op->o_tmpmemctx ); - return SLAP_CB_CONTINUE; -} - -static int -pcache_response( - Operation *op, - SlapReply *rs ) -{ - struct search_info *si = op->o_callback->sc_private; slap_overinst *on = si->on; cache_manager *cm = on->on_bi.bi_private; query_manager* qm = cm->qm; - if ( si->save_attrs != NULL ) { - rs->sr_attrs = si->save_attrs; - op->ors_attrs = si->save_attrs; - } - if ( rs->sr_type == REP_SEARCH ) { Entry *e; + + /* don't return more entries than requested by the client */ + if ( si->slimit && rs->sr_nentries >= si->slimit ) { + si->slimit_exceeded = 1; + } + /* If we haven't exceeded the limit for this query, * build a chain of answers to store. If we hit the * limit, empty the chain and ignore the rest. @@ -1889,6 +1966,7 @@ pcache_response( if ( !si->head ) si->head = e; if ( si->tail ) si->tail->e_private = e; si->tail = e; + } else { si->over = 1; si->count = 0; @@ -1901,16 +1979,47 @@ pcache_response( } } - } else if ( rs->sr_type == REP_RESULT ) { - if ( si->count || - ( si->qtemp->negttl && !si->count && !si->over && - rs->sr_err == LDAP_SUCCESS )) { - CachedQuery *qc = qm->addfunc(op, qm, &si->query, si->qtemp, - si->count); + } + + if ( rs->sr_type == REP_RESULT || + op->o_abandon || rs->sr_err == SLAPD_ABANDON ) + { + if ( si->save_attrs != NULL ) { + rs->sr_attrs = si->save_attrs; + op->ors_attrs = si->save_attrs; + } + if ( (op->o_abandon || rs->sr_err == SLAPD_ABANDON) && + si->caching_reason == PC_IGNORE ) { + filter_free( si->query.filter ); + if ( si->count ) { + /* duplicate query, free it */ + Entry *e; + for (;si->head; si->head=e) { + e = si->head->e_private; + si->head->e_private = NULL; + entry_free(si->head); + } + } + op->o_callback = op->o_callback->sc_next; + op->o_tmpfree( cb, op->o_tmpmemctx ); + } else if ( si->caching_reason != PC_IGNORE ) { + CachedQuery *qc = qm->addfunc(op, qm, &si->query, + si->qtemp, si->caching_reason, 1 ); if ( qc != NULL ) { - if ( si->count ) + switch ( si->caching_reason ) { + case PC_POSITIVE: cache_entries( op, rs, &qc->q_uuid ); + break; + + case PC_SIZELIMIT: + qc->q_sizelimit = rs->sr_nentries; + break; + default: + assert( 0 ); + break; + } + ldap_pvt_thread_rdwr_wunlock(&qc->rwlock); ldap_pvt_thread_mutex_lock(&cm->cache_mutex); cm->num_cached_queries++; Debug( pcache_debug, "STORED QUERIES = %lu\n", @@ -1928,6 +2037,7 @@ pcache_response( } ldap_pvt_thread_mutex_unlock( &slapd_rq.rq_mutex ); } + } else if ( si->count ) { /* duplicate query, free it */ Entry *e; @@ -1937,12 +2047,57 @@ pcache_response( entry_free(si->head); } } + } else { filter_free( si->query.filter ); } + } + + return SLAP_CB_CONTINUE; +} + +static int +pcache_response( + Operation *op, + SlapReply *rs ) +{ + struct search_info *si = op->o_callback->sc_private; + + if ( si->save_attrs != NULL ) { + rs->sr_attrs = si->save_attrs; + op->ors_attrs = si->save_attrs; + } + + if ( rs->sr_type == REP_SEARCH ) { + /* don't return more entries than requested by the client */ + if ( si->slimit_exceeded ) { + return 0; + } + + } else if ( rs->sr_type == REP_RESULT ) { + + if ( si->count ) { + if ( rs->sr_err == LDAP_SUCCESS ) { + si->caching_reason = PC_POSITIVE; + + } else if ( rs->sr_err == LDAP_SIZELIMIT_EXCEEDED + && si->qtemp->limitttl ) + { + si->caching_reason = PC_SIZELIMIT; + } + + } else if ( si->qtemp->negttl && !si->count && !si->over && + rs->sr_err == LDAP_SUCCESS ) + { + si->caching_reason = PC_NEGATIVE; + } - op->o_callback->sc_cleanup = pcache_op_cleanup; + + if ( si->slimit_exceeded ) { + rs->sr_err = LDAP_SIZELIMIT_EXCEEDED; + } } + return SLAP_CB_CONTINUE; } @@ -2060,6 +2215,13 @@ pcache_op_privdb( return SLAP_CB_CONTINUE; } + /* The cache DB isn't open yet */ + if ( cm->defer_db_open ) { + send_ldap_error( op, rs, LDAP_UNAVAILABLE, + "pcachePrivDB: cacheDB not available" ); + return rs->sr_err; + } + /* FIXME: might be a little bit exaggerated... */ if ( !be_isroot( op ) ) { save_cb = op->o_callback; @@ -2075,6 +2237,7 @@ pcache_op_privdb( type = slap_req2op( op->o_tag ); if ( type != SLAP_OP_LAST ) { BI_op_func **func; + int rc; /* execute, if possible */ func = &cm->db.be_bind; @@ -2083,7 +2246,10 @@ pcache_op_privdb( op2.o_bd = &cm->db; - return func[ type ]( &op2, rs ); + rc = func[ type ]( &op2, rs ); + if ( type == SLAP_OP_BIND && rc == LDAP_SUCCESS ) { + op->o_conn->c_authz_cookie = cm->db.be_private; + } } } @@ -2128,6 +2294,13 @@ pcache_op_search( } #endif /* PCACHE_CONTROL_PRIVDB */ + /* The cache DB isn't open yet */ + if ( cm->defer_db_open ) { + send_ldap_error( op, rs, LDAP_UNAVAILABLE, + "pcachePrivDB: cacheDB not available" ); + return rs->sr_err; + } + tempstr.bv_val = op->o_tmpalloc( op->ors_filterstr.bv_len+1, op->o_tmpmemctx ); tempstr.bv_len = 0; if ( filter2template( op, op->ors_filter, &tempstr, &filter_attrs, @@ -2176,6 +2349,7 @@ pcache_op_search( Debug( pcache_debug, "QUERY ANSWERABLE\n", 0, 0, 0 ); op->o_tmpfree( filter_attrs, op->o_tmpmemctx ); + ldap_pvt_thread_rdwr_rlock(&answerable->rwlock); if ( BER_BVISNULL( &answerable->q_uuid )) { /* No entries cached, just an empty result set */ i = rs->sr_err = 0; @@ -2185,6 +2359,7 @@ pcache_op_search( op->o_callback = NULL; i = cm->db.bd_info->bi_op_search( op, rs ); } + ldap_pvt_thread_rdwr_runlock(&answerable->rwlock); ldap_pvt_thread_rdwr_runlock(&qtemp->t_rwlock); op->o_bd = save_bd; op->o_callback = save_cb; @@ -2219,7 +2394,7 @@ pcache_op_search( cb = op->o_tmpalloc( sizeof(*cb) + sizeof(*si), op->o_tmpmemctx ); cb->sc_response = pcache_response; - cb->sc_cleanup = NULL; + cb->sc_cleanup = pcache_op_cleanup; cb->sc_private = (cb+1); si = cb->sc_private; si->on = on; @@ -2228,6 +2403,13 @@ pcache_op_search( si->max = cm->num_entries_limit ; si->over = 0; si->count = 0; + si->slimit = 0; + si->slimit_exceeded = 0; + si->caching_reason = PC_IGNORE; + if ( op->ors_slimit && op->ors_slimit < cm->num_entries_limit ) { + si->slimit = op->ors_slimit; + op->ors_slimit = cm->num_entries_limit; + } si->head = NULL; si->tail = NULL; si->save_attrs = op->ors_attrs; @@ -2341,7 +2523,7 @@ consistency_check( op->o_dn = cm->db.be_rootdn; op->o_ndn = cm->db.be_rootndn; - cm->cc_arg = arg; + cm->cc_arg = arg; for (templ = qm->templates; templ; templ=templ->qmnext) { query = templ->query_last; @@ -2428,9 +2610,10 @@ static ConfigTable pccfg[] = { "DESC 'A set of attributes to cache' " "SYNTAX OMsDirectoryString )", NULL, NULL }, { "proxytemplate", "filter> ca_private; + return pcache_db_open2( on, &c->reply ); +} + static int pc_ldadd( CfEntryInfo *p, Entry *e, ConfigArgs *ca ) { @@ -2479,6 +2680,12 @@ pc_ldadd( CfEntryInfo *p, Entry *e, ConfigArgs *ca ) on = (slap_overinst *)p->ce_bi; cm = on->on_bi.bi_private; ca->be = &cm->db; + /* Defer open if this is an LDAPadd */ + if ( CONFIG_ONLINE_ADD( ca )) + ca->cleanup = pc_ldadd_cleanup; + else + cm->defer_db_open = 0; + ca->ca_private = on; return LDAP_SUCCESS; } @@ -2491,9 +2698,14 @@ pc_cfadd( Operation *op, SlapReply *rs, Entry *p, ConfigArgs *ca ) struct berval bv; /* FIXME: should not hardcode "olcDatabase" here */ - bv.bv_len = sprintf( ca->cr_msg, "olcDatabase=%s", cm->db.bd_info->bi_type ); + bv.bv_len = snprintf( ca->cr_msg, sizeof( ca->cr_msg ), + "olcDatabase=%s", cm->db.bd_info->bi_type ); + if ( bv.bv_len < 0 || bv.bv_len >= sizeof( ca->cr_msg ) ) { + return -1; + } bv.bv_val = ca->cr_msg; ca->be = &cm->db; + cm->defer_db_open = 0; /* We can only create this entry if the database is table-driven */ @@ -2553,17 +2765,14 @@ pc_cf_gen( ConfigArgs *c ) break; case PC_TEMP: for (temp=qm->templates; temp; temp=temp->qmnext) { - if ( temp->negttl ) { - bv.bv_len = snprintf( c->cr_msg, sizeof( c->cr_msg ), - " %d %ld %ld", - temp->attr_set_index, - temp->ttl, - temp->negttl ); - } else { - bv.bv_len = snprintf( c->cr_msg, sizeof( c->cr_msg ), " %d %ld", - temp->attr_set_index, - temp->ttl ); - } + /* HEADS-UP: always print all; + * if optional == 0, ignore */ + bv.bv_len = snprintf( c->cr_msg, sizeof( c->cr_msg ), + " %d %ld %ld %ld", + temp->attr_set_index, + temp->ttl, + temp->negttl, + temp->limitttl ); bv.bv_len += temp->querystr.bv_len + 2; bv.bv_val = ch_malloc( bv.bv_len+1 ); ptr = bv.bv_val; @@ -2666,6 +2875,7 @@ pc_cf_gen( ConfigArgs *c ) Debug( LDAP_DEBUG_CONFIG, "%s: %s.\n", c->log, c->cr_msg, 0 ); return( 1 ); } + cm->cc_period = (time_t)t; Debug( pcache_debug, "Total # of attribute sets to be cached = %d.\n", @@ -2812,23 +3022,37 @@ pc_cf_gen( ConfigArgs *c ) ldap_pvt_thread_rdwr_init( &temp->t_rwlock ); temp->query = temp->query_last = NULL; if ( lutil_parse_time( c->argv[3], &t ) != 0 ) { - snprintf( c->cr_msg, sizeof( c->cr_msg ), "unable to parse template ttl=\"%s\"", + snprintf( c->cr_msg, sizeof( c->cr_msg ), + "unable to parse template ttl=\"%s\"", c->argv[3] ); Debug( LDAP_DEBUG_CONFIG, "%s: %s.\n", c->log, c->cr_msg, 0 ); return( 1 ); } temp->ttl = (time_t)t; - if ( c->argc == 5 ) { + temp->negttl = (time_t)0; + temp->limitttl = (time_t)0; + switch ( c->argc ) { + case 6: + if ( lutil_parse_time( c->argv[5], &t ) != 0 ) { + snprintf( c->cr_msg, sizeof( c->cr_msg ), + "unable to parse template sizelimit ttl=\"%s\"", + c->argv[5] ); + Debug( LDAP_DEBUG_CONFIG, "%s: %s.\n", c->log, c->cr_msg, 0 ); + return( 1 ); + } + temp->limitttl = (time_t)t; + /* fallthru */ + + case 5: if ( lutil_parse_time( c->argv[4], &t ) != 0 ) { snprintf( c->cr_msg, sizeof( c->cr_msg ), - "unable to parse template negttl=\"%s\"", + "unable to parse template negative ttl=\"%s\"", c->argv[4] ); Debug( LDAP_DEBUG_CONFIG, "%s: %s.\n", c->log, c->cr_msg, 0 ); return( 1 ); } temp->negttl = (time_t)t; - } else { - temp->negttl = 0; + break; } temp->no_of_queries = 0; @@ -2919,8 +3143,10 @@ pcache_db_init( cm->max_queries = 10000; cm->save_queries = 0; cm->response_cb = PCACHE_RESPONSE_CB_TAIL; + cm->defer_db_open = 1; cm->cc_period = 1000; cm->cc_paused = 0; + cm->cc_arg = NULL; qm->attr_sets = NULL; qm->templates = NULL; @@ -2975,64 +3201,25 @@ pcache_cachedquery_count_cb( Operation *op, SlapReply *rs ) } static int -pcache_db_open( - BackendDB *be, +pcache_db_open2( + slap_overinst *on, ConfigReply *cr ) { - slap_overinst *on = (slap_overinst *)be->bd_info; cache_manager *cm = on->on_bi.bi_private; query_manager* qm = cm->qm; - int i, ncf = 0, rf = 0, nrf = 0, rc = 0; - - /* check attr sets */ - for ( i = 0; i < cm->numattrsets; i++) { - if ( !( qm->attr_sets[i].flags & PC_CONFIGURED ) ) { - if ( qm->attr_sets[i].flags & PC_REFERENCED ) { - Debug( LDAP_DEBUG_CONFIG, "pcache: attr set #%d not configured but referenced.\n", i, 0, 0 ); - rf++; - - } else { - Debug( LDAP_DEBUG_CONFIG, "pcache: warning, attr set #%d not configured.\n", i, 0, 0 ); - } - ncf++; - - } else if ( !( qm->attr_sets[i].flags & PC_REFERENCED ) ) { - Debug( LDAP_DEBUG_CONFIG, "pcache: attr set #%d configured but not referenced.\n", i, 0, 0 ); - nrf++; - } - } - - if ( ncf || rf || nrf ) { - Debug( LDAP_DEBUG_CONFIG, "pcache: warning, %d attr sets configured but not referenced.\n", nrf, 0, 0 ); - Debug( LDAP_DEBUG_CONFIG, "pcache: warning, %d attr sets not configured.\n", ncf, 0, 0 ); - Debug( LDAP_DEBUG_CONFIG, "pcache: %d attr sets not configured but referenced.\n", rf, 0, 0 ); - - if ( rf > 0 ) { - return 1; - } - } - - /* need to inherit something from the original database... */ - cm->db.be_def_limit = be->be_def_limit; - cm->db.be_limits = be->be_limits; - cm->db.be_acl = be->be_acl; - cm->db.be_dfltaccess = be->be_dfltaccess; - - if ( SLAP_DBMONITORING( be ) ) { - SLAP_DBFLAGS( &cm->db ) |= SLAP_DBFLAG_MONITORING; - - } else { - SLAP_DBFLAGS( &cm->db ) &= ~SLAP_DBFLAG_MONITORING; - } + int rc; rc = backend_startup_one( &cm->db, NULL ); + if ( rc == 0 ) { + cm->defer_db_open = 0; + } /* There is no runqueue in TOOL mode */ - if ( slapMode & SLAP_SERVER_MODE ) { + if (( slapMode & SLAP_SERVER_MODE ) && rc == 0 ) { ldap_pvt_thread_mutex_lock( &slapd_rq.rq_mutex ); ldap_pvt_runqueue_insert( &slapd_rq, cm->cc_period, consistency_check, on, - "pcache_consistency", be->be_suffix[0].bv_val ); + "pcache_consistency", cm->db.be_suffix[0].bv_val ); ldap_pvt_thread_mutex_unlock( &slapd_rq.rq_mutex ); /* Cached database must have the rootdn */ @@ -3057,11 +3244,7 @@ pcache_db_open( SlapReply rs = { 0 }; BerVarray vals = NULL; Filter f = { 0 }, f2 = { 0 }; -#ifdef LDAP_COMP_MATCH - AttributeAssertion ava = { NULL, BER_BVNULL, NULL }; -#else - AttributeAssertion ava = { NULL, BER_BVNULL }; -#endif + AttributeAssertion ava = ATTRIBUTEASSERTION_INIT; AttributeName attrs[ 2 ] = { 0 }; connection_fake_init( &conn, &opbuf, thrctx ); @@ -3134,6 +3317,62 @@ pcache_db_open( rc = 0; } } + return rc; +} + +static int +pcache_db_open( + BackendDB *be, + ConfigReply *cr ) +{ + slap_overinst *on = (slap_overinst *)be->bd_info; + cache_manager *cm = on->on_bi.bi_private; + query_manager* qm = cm->qm; + int i, ncf = 0, rf = 0, nrf = 0, rc = 0; + + /* check attr sets */ + for ( i = 0; i < cm->numattrsets; i++) { + if ( !( qm->attr_sets[i].flags & PC_CONFIGURED ) ) { + if ( qm->attr_sets[i].flags & PC_REFERENCED ) { + Debug( LDAP_DEBUG_CONFIG, "pcache: attr set #%d not configured but referenced.\n", i, 0, 0 ); + rf++; + + } else { + Debug( LDAP_DEBUG_CONFIG, "pcache: warning, attr set #%d not configured.\n", i, 0, 0 ); + } + ncf++; + + } else if ( !( qm->attr_sets[i].flags & PC_REFERENCED ) ) { + Debug( LDAP_DEBUG_CONFIG, "pcache: attr set #%d configured but not referenced.\n", i, 0, 0 ); + nrf++; + } + } + + if ( ncf || rf || nrf ) { + Debug( LDAP_DEBUG_CONFIG, "pcache: warning, %d attr sets configured but not referenced.\n", nrf, 0, 0 ); + Debug( LDAP_DEBUG_CONFIG, "pcache: warning, %d attr sets not configured.\n", ncf, 0, 0 ); + Debug( LDAP_DEBUG_CONFIG, "pcache: %d attr sets not configured but referenced.\n", rf, 0, 0 ); + + if ( rf > 0 ) { + return 1; + } + } + + /* need to inherit something from the original database... */ + cm->db.be_def_limit = be->be_def_limit; + cm->db.be_limits = be->be_limits; + cm->db.be_acl = be->be_acl; + cm->db.be_dfltaccess = be->be_dfltaccess; + + if ( SLAP_DBMONITORING( be ) ) { + SLAP_DBFLAGS( &cm->db ) |= SLAP_DBFLAG_MONITORING; + + } else { + SLAP_DBFLAGS( &cm->db ) &= ~SLAP_DBFLAG_MONITORING; + } + + if ( !cm->defer_db_open ) + rc = pcache_db_open2( on, cr ); return rc; } @@ -3212,6 +3451,7 @@ pcache_db_close( mod.sml_type = ad_cachedQueryURL->ad_cname; mod.sml_values = vals; mod.sml_nvalues = NULL; + mod.sml_numvals = 1; mod.sml_next = NULL; Debug( pcache_debug, "%sSETTING CACHED QUERY URLS\n", @@ -3542,37 +3782,35 @@ pcache_exop_query_delete( { BackendDB *bd = op->o_bd; - struct berval uuid = BER_BVNULL; + struct berval uuid = BER_BVNULL, + *uuidp = NULL; char buf[ SLAP_TEXT_BUFLEN ] = { '\0' }; int len = 0; ber_tag_t tag = LBER_DEFAULT; + if ( LogTest( LDAP_DEBUG_STATS ) ) { + uuidp = &uuid; + } + rs->sr_err = pcache_parse_query_delete( op->ore_reqdata, - &tag, &op->o_req_ndn, &uuid, + &tag, &op->o_req_ndn, uuidp, &rs->sr_text, op->o_tmpmemctx ); if ( rs->sr_err != LDAP_SUCCESS ) { return rs->sr_err; } if ( LogTest( LDAP_DEBUG_STATS ) ) { - if ( !BER_BVISNULL( &op->o_req_ndn ) ) { - len = snprintf( buf, sizeof( buf ), " dn=\"%s\"", op->o_req_ndn.bv_val ); - } + assert( !BER_BVISNULL( &op->o_req_ndn ) ); + len = snprintf( buf, sizeof( buf ), " dn=\"%s\"", op->o_req_ndn.bv_val ); if ( !BER_BVISNULL( &uuid ) ) { - char uuidbuf[ LDAP_LUTIL_UUIDSTR_BUFSIZE ]; - - lutil_uuidstr_from_normalized( - uuid.bv_val, uuid.bv_len, - uuidbuf, sizeof( uuidbuf ) ); - - snprintf( &buf[ len ], sizeof( buf ) - len, " UUID=\"%s\"", uuidbuf ); + snprintf( &buf[ len ], sizeof( buf ) - len, " queryId=\"%s\"", uuid.bv_val ); } Debug( LDAP_DEBUG_STATS, "%s QUERY DELETE%s\n", op->o_log_prefix, buf, 0 ); - op->o_req_dn = op->o_req_ndn; } + op->o_req_dn = op->o_req_ndn; op->o_bd = select_backend( &op->o_req_ndn, 0 ); rs->sr_err = backend_check_restrictions( op, rs, @@ -3633,9 +3871,8 @@ pcache_op_extended( Operation *op, SlapReply *rs ) } else if ( tag == LDAP_TAG_EXOP_QUERY_DELETE_BASE ) { if ( !BER_BVISNULL( &uuid ) ) { /* remove the selected query */ - remove_query_and_data( op, rs, cm, &uuid ); - op->o_tmpfree( uuid.bv_val, op->o_tmpmemctx ); - rs->sr_err = LDAP_SUCCESS; + rs->sr_err = pcache_remove_query_from_cache( op, + cm, &uuid ); } else { /* TODO: remove all queries */ @@ -3643,6 +3880,8 @@ pcache_op_extended( Operation *op, SlapReply *rs ) rs->sr_text = "deletion of all queries not implemented"; } } + + op->o_tmpfree( uuid.bv_val, op->o_tmpmemctx ); } return rs->sr_err; @@ -3701,6 +3940,7 @@ pcache_initialize() "pcache_initialize: register_at #%d failed\n", i, 0, 0 ); return code; } + (*as[i].adp)->ad_type->sat_flags |= SLAP_AT_HIDE; } pcache.on_bi.bi_type = "pcache"; @@ -3733,6 +3973,11 @@ pcache_initialize() code = config_register_schema( pccfg, pcocs ); if ( code ) return code; + { + const char *text; + code = slap_str2ad( "olcDatabase", &pcdummy[0].ad, &text ); + if ( code ) return code; + } return overlay_register( &pcache ); }