From 6f81235386942d37260f70a7d898865fc5f1ccd1 Mon Sep 17 00:00:00 2001 From: Jong Hyuk Choi Date: Thu, 6 Mar 2003 19:44:41 +0000 Subject: [PATCH] Commit of the Proxy Cache contribution (ITS#2062) from IBM India Research (Apurva Kumar) Code review by Pierangelo, Kurt, and Jong. --- servers/slapd/back-ldbm/add.c | 42 +- servers/slapd/back-ldbm/delete.c | 8 +- servers/slapd/back-ldbm/modify.c | 33 +- servers/slapd/back-ldbm/search.c | 78 +- servers/slapd/back-meta/Makefile.in | 4 + servers/slapd/back-meta/back-meta.h | 8 +- servers/slapd/back-meta/cache-config.c | 332 ++++ servers/slapd/back-meta/cache-merge.c | 408 +++++ servers/slapd/back-meta/cache-query.c | 462 ++++++ servers/slapd/back-meta/cache-remove.c | 183 +++ servers/slapd/back-meta/cache-search.c | 1724 +++++++++++++++++++++ servers/slapd/back-meta/cache-substring.c | 290 ++++ servers/slapd/back-meta/cache-template.c | 247 +++ servers/slapd/back-meta/cache.h | 252 +++ servers/slapd/back-meta/config.c | 16 +- servers/slapd/back-meta/init.c | 58 + servers/slapd/back-meta/search.c | 10 + servers/slapd/schema_prep.c | 12 + servers/slapd/slap.h | 9 + 19 files changed, 4156 insertions(+), 20 deletions(-) create mode 100644 servers/slapd/back-meta/cache-config.c create mode 100644 servers/slapd/back-meta/cache-merge.c create mode 100644 servers/slapd/back-meta/cache-query.c create mode 100644 servers/slapd/back-meta/cache-remove.c create mode 100644 servers/slapd/back-meta/cache-search.c create mode 100644 servers/slapd/back-meta/cache-substring.c create mode 100644 servers/slapd/back-meta/cache-template.c create mode 100644 servers/slapd/back-meta/cache.h diff --git a/servers/slapd/back-ldbm/add.c b/servers/slapd/back-ldbm/add.c index 560ef7baea..ff115aa711 100644 --- a/servers/slapd/back-ldbm/add.c +++ b/servers/slapd/back-ldbm/add.c @@ -40,8 +40,17 @@ ldbm_back_add( #else Debug(LDAP_DEBUG_ARGS, "==> ldbm_back_add: %s\n", e->e_dn, 0, 0); #endif - + +#ifndef LDAP_CACHING rc = entry_schema_check( be, e, NULL, &text, textbuf, textlen ); +#else /* LDAP_CACHING */ + if ( !op->o_caching_on ) { + rc = entry_schema_check( be, e, NULL, &text, textbuf, textlen ); + } else { + rc = LDAP_SUCCESS; + } +#endif /* LDAP_CACHING */ + if ( rc != LDAP_SUCCESS ) { #ifdef NEW_LOGGING LDAP_LOG( BACK_LDBM, ERR, @@ -56,8 +65,11 @@ ldbm_back_add( return( -1 ); } - if ( ! access_allowed( be, conn, op, e, - entry, NULL, ACL_WRITE, NULL ) ) +#ifdef LDAP_CACHING + if ( !op->o_caching_on ) { +#endif /* LDAP_CACHING */ + if ( !access_allowed( be, conn, op, e, + entry, NULL, ACL_WRITE, NULL ) ) { #ifdef NEW_LOGGING LDAP_LOG( BACK_LDBM, ERR, @@ -73,6 +85,9 @@ ldbm_back_add( return -1; } +#ifdef LDAP_CACHING + } +#endif /* LDAP_CACHING */ /* grab giant lock for writing */ ldap_pvt_thread_rdwr_wlock(&li->li_giant_rwlock); @@ -98,7 +113,12 @@ ldbm_back_add( dnParent( &e->e_nname, &pdn ); } - if( pdn.bv_len ) { +#ifndef LDAP_CACHING + if( pdn.bv_len ) +#else /* LDAP_CACHING */ + if( pdn.bv_len && !op->o_caching_on ) +#endif /* LDAP_CACHING */ + { Entry *matched = NULL; /* get parent with writer lock */ @@ -211,12 +231,22 @@ ldbm_back_add( } } else { - if(pdn.bv_val != NULL) { +#ifndef LDAP_CACHING + if( pdn.bv_val != NULL ) +#else /* LDAP_CACHING */ + if( pdn.bv_val != NULL && !op->o_caching_on ) +#endif /* LDAP_CACHING */ + { assert( *pdn.bv_val == '\0' ); } /* no parent, must be adding entry to root */ - if ( !be_isroot( be, &op->o_ndn ) ) { +#ifndef LDAP_CACHING + if ( !be_isroot( be, &op->o_ndn ) ) +#else /* LDAP_CACHING */ + if ( !be_isroot( be, &op->o_ndn ) && !op->o_caching_on ) +#endif /* LDAP_CACHING */ + { if ( be_issuffix( be, (struct berval *)&slap_empty_bv ) || be_isupdate( be, &op->o_ndn ) ) { p = (Entry *)&slap_entry_root; diff --git a/servers/slapd/back-ldbm/delete.c b/servers/slapd/back-ldbm/delete.c index 40a0925ad2..9f746fb2a7 100644 --- a/servers/slapd/back-ldbm/delete.c +++ b/servers/slapd/back-ldbm/delete.c @@ -80,6 +80,9 @@ ldbm_back_delete( } /* check entry for "entry" acl */ +#ifdef LDAP_CACHING + if( !op->o_caching_on ) { +#endif /* LDAP_CACHING */ if ( ! access_allowed( be, conn, op, e, entry, NULL, ACL_WRITE, NULL ) ) { @@ -100,7 +103,7 @@ ldbm_back_delete( goto return_results; } - if ( !manageDSAit && is_entry_referral( e ) ) { + if ( !manageDSAit && is_entry_referral( e ) ) { /* parent is a referral, don't allow add */ /* parent is an alias, don't allow add */ BerVarray refs = get_entry_referrals( be, @@ -219,6 +222,9 @@ ldbm_back_delete( } } } +#ifdef LDAP_CACHING + } +#endif /* LDAP_CACHING */ /* delete from dn2id mapping */ if ( dn2id_delete( be, &e->e_nname, e->e_id ) != 0 ) { diff --git a/servers/slapd/back-ldbm/modify.c b/servers/slapd/back-ldbm/modify.c index b7bd5c55e7..717c4691b3 100644 --- a/servers/slapd/back-ldbm/modify.c +++ b/servers/slapd/back-ldbm/modify.c @@ -65,8 +65,8 @@ int ldbm_modify_internal( Debug(LDAP_DEBUG_ARGS, "ldbm_modify_internal: add\n", 0, 0, 0); #endif - rc = modify_add_values( e, mod, get_permissiveModify(op), - text, textbuf, textlen ); + rc = modify_add_values( e, mod, get_permissiveModify( op ), + text, textbuf, textlen ); if( rc != LDAP_SUCCESS ) { #ifdef NEW_LOGGING LDAP_LOG( BACK_LDBM, INFO, @@ -85,8 +85,8 @@ int ldbm_modify_internal( Debug(LDAP_DEBUG_ARGS, "ldbm_modify_internal: delete\n", 0, 0, 0); #endif - rc = modify_delete_values( e, mod, get_permissiveModify(op), - text, textbuf, textlen ); + rc = modify_delete_values( e, mod, get_permissiveModify( op ), + text, textbuf, textlen ); assert( rc != LDAP_TYPE_OR_VALUE_EXISTS ); if( rc != LDAP_SUCCESS ) { #ifdef NEW_LOGGING @@ -106,8 +106,8 @@ int ldbm_modify_internal( Debug(LDAP_DEBUG_ARGS, "ldbm_modify_internal: replace\n", 0, 0, 0); #endif - rc = modify_replace_values( e, mod, get_permissiveModify(op), - text, textbuf, textlen ); + rc = modify_replace_values( e, mod, get_permissiveModify( op ), + text, textbuf, textlen ); if( rc != LDAP_SUCCESS ) { #ifdef NEW_LOGGING LDAP_LOG( BACK_LDBM, INFO, @@ -132,8 +132,8 @@ int ldbm_modify_internal( */ mod->sm_op = LDAP_MOD_ADD; - rc = modify_add_values( e, mod, get_permissiveModify(op), - text, textbuf, textlen ); + rc = modify_add_values( e, mod, get_permissiveModify( op ), + text, textbuf, textlen ); if ( rc == LDAP_TYPE_OR_VALUE_EXISTS ) { rc = LDAP_SUCCESS; } @@ -190,7 +190,17 @@ int ldbm_modify_internal( } /* check that the entry still obeys the schema */ +#ifndef LDAP_CACHING rc = entry_schema_check( be, e, save_attrs, text, textbuf, textlen ); +#else /* LDAP_CACHING */ + if ( !op->o_caching_on ) { + rc = entry_schema_check( be, e, save_attrs, + text, textbuf, textlen ); + } else { + rc = LDAP_SUCCESS; + } +#endif /* LDAP_CACHING */ + if ( rc != LDAP_SUCCESS ) { #ifdef NEW_LOGGING LDAP_LOG( BACK_LDBM, ERR, @@ -322,7 +332,12 @@ ldbm_back_modify( return( -1 ); } - if ( !manageDSAit && is_entry_referral( e ) ) { +#ifndef LDAP_CACHING + if ( !manageDSAit && is_entry_referral( e ) ) +#else /* LDAP_CACHING */ + if ( !op->o_caching_on && !manageDSAit && is_entry_referral( e ) ) +#endif /* LDAP_CACHING */ + { /* parent is a referral, don't allow add */ /* parent is an alias, don't allow add */ BerVarray refs = get_entry_referrals( be, diff --git a/servers/slapd/back-ldbm/search.c b/servers/slapd/back-ldbm/search.c index 12ec9a6c01..8a1a2604ae 100644 --- a/servers/slapd/back-ldbm/search.c +++ b/servers/slapd/back-ldbm/search.c @@ -54,6 +54,10 @@ ldbm_back_search( int manageDSAit = get_manageDSAit( op ); int cscope = LDAP_SCOPE_DEFAULT; +#ifdef LDAP_CACHING + Entry cache_base_entry; +#endif /* LDAP_CACHING */ + struct slap_limits_set *limit = NULL; int isroot = 0; @@ -66,6 +70,7 @@ ldbm_back_search( /* grab giant lock for reading */ ldap_pvt_thread_rdwr_rlock(&li->li_giant_rwlock); +#ifndef LDAP_CACHING if ( nbase->bv_len == 0 ) { /* DIT root special case */ e = (Entry *) &slap_entry_root; @@ -73,8 +78,28 @@ ldbm_back_search( /* need normalized dn below */ ber_dupbv( &realbase, &e->e_nname ); +#else /* LDAP_CACHING */ + if ( op->o_caching_on || nbase->bv_len == 0 ) { + if (nbase->bv_len == 0) { + e = (Entry *) &slap_entry_root; + /* need normalized dn below */ + ber_dupbv( &realbase, &e->e_nname ); + } else { + if ((scope == LDAP_SCOPE_BASE) + && (e = dn2entry_r( be, nbase, &matched ))) + { + candidates = base_candidate(be,e); + cache_return_entry_r( &li->li_cache, e ); + goto searchit; + } + cache_base_entry.e_nname = *nbase; + e = &cache_base_entry; + } +#endif /* LDAP_CACHING */ + candidates = search_candidates( be, e, filter, - scope, deref, manageDSAit || get_domainScope(op) ); + scope, deref, + manageDSAit || get_domainScope(op) ); goto searchit; @@ -203,17 +228,37 @@ searchit: Debug( LDAP_DEBUG_TRACE, "ldbm_search: no candidates\n", 0, 0, 0 ); #endif +#ifdef LDAP_CACHING + if ( op->o_caching_on ) { + ldap_pvt_thread_rdwr_runlock(&li->li_giant_rwlock); + } +#endif /* LDAP_CACHING */ send_search_result( conn, op, LDAP_SUCCESS, NULL, NULL, NULL, NULL, 0 ); +#ifdef LDAP_CACHING + if ( op->o_caching_on ) { + ldap_pvt_thread_rdwr_rlock(&li->li_giant_rwlock); + } +#endif /* LDAP_CACHING */ + rc = 1; goto done; } /* if not root, get appropriate limits */ - if ( be_isroot( be, &op->o_ndn ) ) { +#ifndef LDAP_CACHING + if ( be_isroot( be, &op->o_ndn ) ) +#else /* LDAP_CACHING */ + if ( op->o_caching_on || be_isroot( be, &op->o_ndn ) ) +#endif /* LDAP_CACHING */ + { + /* + * FIXME: I'd consider this dangerous if someone + * uses isroot for anything but handling limits + */ isroot = 1; } else { ( void ) get_limits( be, &op->o_ndn, &limit ); @@ -328,6 +373,10 @@ searchit: goto loop_continue; } +#ifdef LDAP_CACHING + if ( !op->o_caching_on ) { +#endif /* LDAP_CACHING */ + if ( deref & LDAP_DEREF_SEARCHING && is_entry_alias( e ) ) { Entry *matched; int err; @@ -425,6 +474,10 @@ searchit: goto loop_continue; } +#ifdef LDAP_CACHING + } +#endif /* LDAP_CACHING */ + /* if it matches the filter and scope, send it */ result = test_filter( be, conn, op, e, filter ); @@ -459,9 +512,24 @@ searchit: } if (e) { + +#ifdef LDAP_CACHING + if ( op->o_caching_on ) { + ldap_pvt_thread_rdwr_runlock(&li->li_giant_rwlock); + cache_return_entry_r( &li->li_cache, e ); + } +#endif /* LDAP_CACHING */ + result = send_search_entry(be, conn, op, e, attrs, attrsonly, NULL); +#ifdef LDAP_CACHING + if ( op->o_caching_on ) { + ldap_pvt_thread_rdwr_rlock( &li->li_giant_rwlock ); + } +#endif /* LDAP_CACHING */ + + switch (result) { case 0: /* entry sent ok */ nentries++; @@ -501,7 +569,13 @@ searchit: loop_continue: if( e != NULL ) { /* free reader lock */ +#ifndef LDAP_CACHING cache_return_entry_r( &li->li_cache, e ); +#else /* LDAP_CACHING */ + if ( !op->o_caching_on ) { + cache_return_entry_r( &li->li_cache, e ); + } +#endif /* LDAP_CACHING */ } ldap_pvt_thread_yield(); diff --git a/servers/slapd/back-meta/Makefile.in b/servers/slapd/back-meta/Makefile.in index da325d42a3..e275e2e37e 100644 --- a/servers/slapd/back-meta/Makefile.in +++ b/servers/slapd/back-meta/Makefile.in @@ -1,9 +1,13 @@ # $OpenLDAP$ SRCS = init.c config.c search.c bind.c unbind.c add.c compare.c \ + cache-query.c cache-search.c cache-config.c cache-merge.c \ + cache-template.c cache-substring.c cache-remove.c \ delete.c modify.c modrdn.c group.c attribute.c \ conn.c candidates.c dncache.c OBJS = init.lo config.lo search.lo bind.lo unbind.lo add.lo compare.lo \ + cache-query.lo cache-search.lo cache-config.lo cache-merge.lo \ + cache-template.lo cache-substring.lo cache-remove.lo \ delete.lo modify.lo modrdn.lo group.lo attribute.lo \ conn.lo candidates.lo dncache.lo diff --git a/servers/slapd/back-meta/back-meta.h b/servers/slapd/back-meta/back-meta.h index 6b02654da0..bcd1e9990e 100644 --- a/servers/slapd/back-meta/back-meta.h +++ b/servers/slapd/back-meta/back-meta.h @@ -77,7 +77,7 @@ /* String rewrite library */ #include "rewrite.h" - +#include "cache.h" LDAP_BEGIN_DECL struct slap_conn; @@ -146,6 +146,12 @@ struct metainfo { #define META_DEFAULT_TARGET_NONE -1 struct metatarget **targets; +#ifdef LDAP_CACHING + struct rewrite_info *rwinfo; + cache_manager *cm; + Backend *glue_be; +#endif /* LDAP_CACHING */ + struct metadncache cache; ldap_pvt_thread_mutex_t conn_mutex; diff --git a/servers/slapd/back-meta/cache-config.c b/servers/slapd/back-meta/cache-config.c new file mode 100644 index 0000000000..30f99a4d27 --- /dev/null +++ b/servers/slapd/back-meta/cache-config.c @@ -0,0 +1,332 @@ +/* Copyright (c) 2003 by International Business Machines, Inc. + * + * International Business Machines, Inc. (hereinafter called IBM) grants + * permission under its copyrights to use, copy, modify, and distribute this + * Software with or without fee, provided that the above copyright notice and + * all paragraphs of this notice appear in all copies, and that the name of IBM + * not be used in connection with the marketing of any product incorporating + * the Software or modifications thereof, without specific, written prior + * permission. + * + * THE SOFTWARE IS PROVIDED "AS IS", AND IBM DISCLAIMS ALL WARRANTIES, + * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A + * PARTICULAR PURPOSE. IN NO EVENT SHALL IBM BE LIABLE FOR ANY SPECIAL, + * DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER ARISING + * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE, EVEN + * IF IBM IS APPRISED OF THE POSSIBILITY OF SUCH DAMAGES. + */ + +#include "portable.h" + +#include + +#include +#include + +#include "slap.h" +#include "../back-ldap/back-ldap.h" +#include "back-meta.h" + +#ifdef LDAP_CACHING +#define MAX_ATTR_SETS 500 +static void find_supersets( struct attr_set* attr_sets, int numsets ); +static int compare_sets( struct attr_set* setA, int, int ); + +int +meta_back_cache_config( + BackendDB *be, + const char *fname, + int lineno, + int argc, + char **argv +) +{ + struct metainfo *li = ( struct metainfo * )be->be_private; + + cache_manager* cm = li->cm; + query_manager* qm = cm->qm; + QueryTemplate* temp; + AttributeName* attr_name; + AttributeName* attrs; + AttributeName* attrarray; + const char* text=NULL; + + int index, i; + int num; + + if ( li == NULL ) { + fprintf( stderr, "%s: line %d: meta backend info is null!\n", + fname, lineno ); + return 1; + } + + if ( strcasecmp( argv[0], "cacheparams" ) == 0 ) { + struct berval cache_suffix; + + if ( argc < 6 ) { + fprintf( stderr, "%s: line %d: missing arguments in \"cacheparams" + " " + "\" \n", fname, lineno ); + return( 1 ); + } + + cm->caching = 1; + cm->thresh_lo = atoi( argv[1] ); + cm->thresh_hi = atoi( argv[2] ); + if ( cm->thresh_hi <= cm->thresh_lo ) { + fprintf( stderr, "%s: line %d: must be < " + "in \"cacheparams " + " \" \n", fname, lineno ); + return( 1 ); + } + + cm->numattrsets = atoi( argv[3] ); + if ( cm->numattrsets > MAX_ATTR_SETS ) { + fprintf( stderr, "%s: line %d: must be <= %d in " + "\"cacheparams " + " \" \n", + fname, lineno, MAX_ATTR_SETS ); + return( 1 ); + } + + cm->num_entries_limit = atoi( argv[4] ); + cm->consistency_cycle_time = atoi( argv[5] ); +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, + "Total # of attribute sets to be cached = %d\n", + cm->numattrsets, 0, 0 ); +#else + Debug( LDAP_DEBUG_ANY, + "Total # of attribute sets to be cached = %d\n", + cm->numattrsets, 0, 0 ); +#endif + qm->attr_sets = ( struct attr_set * )malloc( cm->numattrsets * + sizeof( struct attr_set )); + for ( i = 0; i < cm->numattrsets; i++ ) { + qm->attr_sets[i].attrs = NULL; + } + rewrite_session( li->rwinfo, "cacheBase", be->be_nsuffix[0].bv_val, + 0, &cache_suffix.bv_val ); + if ( cache_suffix.bv_val != NULL ) { + cache_suffix.bv_len = strlen( cache_suffix.bv_val ); + } else { + cache_suffix = be->be_nsuffix[0]; + } + li->glue_be = select_backend( &cache_suffix, 0, 1 ); + if ( cache_suffix.bv_val != be->be_nsuffix[0].bv_val ) { + ch_free( cache_suffix.bv_val ); + } + + } else if ( strcasecmp( argv[0], "attrset" ) == 0 ) { + if ( argc < 3 ) { + fprintf( stderr, "%s: line %d: missing arguments in \"attr-set " + " \" line\n", fname, lineno ); + return( 1 ); + } +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, "Attribute Set # %d\n", + atoi( argv[1] ), 0, 0 ); +#else + Debug( LDAP_DEBUG_ANY, "Attribute Set # %d\n", + atoi( argv[1] ), 0, 0 ); +#endif + if (atoi(argv[1]) >= cm->numattrsets) { + fprintf( stderr, "%s; line %d index out of bounds \n", + fname, lineno ); + return 1; + } + index = atoi( argv[1] ); + if ( argv[2] && ( strcmp( argv[2], "*" ) != 0 )) { + for ( i = 2; argv[i] != NULL; i++ ) { +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, "\t %s\n", + argv[i], 0, 0 ); +#else + Debug( LDAP_DEBUG_ANY, "\t %s\n", + argv[i], 0, 0 ); +#endif + attrs = qm->attr_sets[index].attrs; + qm->attr_sets[index].attrs = (AttributeName*)realloc( + attrs, i * sizeof( AttributeName )); + attr_name = qm->attr_sets[index].attrs + ( i - 2 ); + ber_str2bv( argv[i], strlen(argv[i]), 1, + &attr_name->an_name); + attr_name->an_desc = NULL; + slap_bv2ad( &attr_name->an_name, + &attr_name->an_desc, &text ); + attr_name++; + attr_name->an_name.bv_val = NULL; + attr_name->an_name.bv_len = 0; + } + qm->attr_sets[index].count = i - 2; + } + } else if ( strcasecmp( argv[0], "addtemplate" ) == 0 ) { + if ( argc != 4 ) { + fprintf( stderr, "%s: line %d: missing argument(s) in " + "\"addtemplate \" line\n", + fname, lineno ); + return( 1 ); + } + if (( i = atoi( argv[2] )) >= cm->numattrsets ) { +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, + "%s: line %d, template index invalid\n", + fname, lineno, 0 ); +#else + Debug( LDAP_DEBUG_ANY, + "%s: line %d, template index invalid\n", + fname, lineno, 0 ); +#endif + return 1; + } + num = cm->numtemplates; + if ( num == 0 ) + find_supersets( qm->attr_sets, cm->numattrsets ); + qm->templates = ( QueryTemplate* )realloc( qm->templates, + ( num + 2 ) * sizeof( QueryTemplate )); + temp = qm->templates + num; + ldap_pvt_thread_rdwr_init( &temp->t_rwlock ); + temp->query = temp->query_last = NULL; + temp->ttl = atoi( argv[3] ); + temp->no_of_queries = 0; + if ( argv[1] == NULL ) { +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, + "Templates string not specified " + "for template %d\n", num, 0, 0 ); +#else + Debug( LDAP_DEBUG_ANY, + "Templates string not specified " + "for template %d\n", num, 0, 0 ); +#endif + return 1; + } + temp->querystr = ch_strdup( argv[1] ); +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, "Template:\n", 0, 0, 0 ); +#else + Debug( LDAP_DEBUG_ANY, "Template:\n", 0, 0, 0 ); +#endif +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, " query template: %s\n", + temp->querystr, 0, 0 ); +#else + Debug( LDAP_DEBUG_ANY, " query template: %s\n", + temp->querystr, 0, 0 ); +#endif + temp->attr_set_index = i; +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, " attributes: \n", 0, 0, 0 ); +#else + Debug( LDAP_DEBUG_ANY, " attributes: \n", 0, 0, 0 ); +#endif + if ( attrarray = qm->attr_sets[i].attrs ) { + for ( i=0; attrarray[i].an_name.bv_val; i++ ) +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, "\t%s\n", + attrarray[i].an_name.bv_val, 0, 0 ); +#else + Debug( LDAP_DEBUG_ANY, "\t%s\n", + attrarray[i].an_name.bv_val, 0, 0 ); +#endif + } + temp++; + temp->querystr = NULL; + cm->numtemplates++; + } + /* anything else */ + else { + fprintf( stderr, "%s: line %d: unknown directive \"%s\" in meta " + "database definition (ignored)\n", + fname, lineno, argv[0] ); + } + return 0; +} + +void +find_supersets ( struct attr_set* attr_sets, int numsets ) +{ + int num[MAX_ATTR_SETS]; + int i, j, res; + int* id_array; + for ( i = 0; i < MAX_ATTR_SETS; i++ ) + num[i] = 0; + + for ( i = 0; i < numsets; i++ ) { + attr_sets[i].ID_array = (int*) malloc( sizeof( int ) ); + attr_sets[i].ID_array[0] = -1; + } + + for ( i = 0; i < numsets; i++ ) { + for ( j=i+1; j < numsets; j++ ) { + res = compare_sets( attr_sets, i, j ); + switch ( res ) { + case 0: + break; + case 3: + case 1: + id_array = attr_sets[i].ID_array; + attr_sets[i].ID_array = (int *) realloc( id_array, + ( num[i] + 2 ) * sizeof( int )); + attr_sets[i].ID_array[num[i]] = j; + attr_sets[i].ID_array[num[i]+1] = -1; + num[i]++; + if (res == 1) + break; + case 2: + id_array = attr_sets[j].ID_array; + attr_sets[j].ID_array = (int *) realloc( id_array, + ( num[j] + 2 ) * sizeof( int )); + attr_sets[j].ID_array[num[j]] = i; + attr_sets[j].ID_array[num[j]+1] = -1; + num[j]++; + break; + } + } + } +} + +/* + * compares two sets of attributes (indices i and j) + * returns 0: if neither set is contained in the other set + * 1: if set i is contained in set j + * 2: if set j is contained in set i + * 3: the sets are equivalent + */ + +int +compare_sets(struct attr_set* set, int i, int j) +{ + int k,l,numI,numJ; + int common=0; + int result=0; + + if (( set[i].attrs == NULL ) && ( set[j].attrs == NULL )) + return 3; + + if ( set[i].attrs == NULL ) + return 2; + + if ( set[j].attrs == NULL ) + return 1; + + numI = set[i].count; + numJ = set[j].count; + + for ( l=0; l < numI; l++ ) { + for ( k = 0; k < numJ; k++ ) { + if ( strcmp( set[i].attrs[l].an_name.bv_val, + set[j].attrs[k].an_name.bv_val ) == 0 ) + common++; + } + } + + if ( common == numI ) + result = 1; + + if ( common == numJ ) + result += 2; + + return result; +} +#endif diff --git a/servers/slapd/back-meta/cache-merge.c b/servers/slapd/back-meta/cache-merge.c new file mode 100644 index 0000000000..8c453e11d6 --- /dev/null +++ b/servers/slapd/back-meta/cache-merge.c @@ -0,0 +1,408 @@ +/* Copyright (c) 2003 by International Business Machines, Inc. + * + * International Business Machines, Inc. (hereinafter called IBM) grants + * permission under its copyrights to use, copy, modify, and distribute this + * Software with or without fee, provided that the above copyright notice and + * all paragraphs of this notice appear in all copies, and that the name of IBM + * not be used in connection with the marketing of any product incorporating + * the Software or modifications thereof, without specific, written prior + * permission. + * + * THE SOFTWARE IS PROVIDED "AS IS", AND IBM DISCLAIMS ALL WARRANTIES, + * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A + * PARTICULAR PURPOSE. IN NO EVENT SHALL IBM BE LIABLE FOR ANY SPECIAL, + * DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER ARISING + * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE, EVEN + * IF IBM IS APPRISED OF THE POSSIBILITY OF SUCH DAMAGES. + */ + +#include "portable.h" + +#include + +#include +#include +#include + +#include "slap.h" +#include "ldif.h" +#include "../back-ldap/back-ldap.h" +#include "back-meta.h" +#include "ldap_pvt.h" +#undef ldap_debug /* silence a warning in ldap-int.h */ +#include "ldap_log.h" +#include "../../../libraries/libldap/ldap-int.h" +#include + +#ifdef LDAP_CACHING +static int +merge_func ( + Backend *be, + Connection *conn, + Operation *op, + Entry *stored_entry, + AttributeName *attrs, + int attrsonly, + LDAPControl **ctrls +); + +void +add_func ( + Connection *conn, + Operation *op, + ber_int_t err, + const char *matched, + const char *text, + BerVarray refs, + LDAPControl **ctrls, + int nentries +); + +static Attribute* +add_attribute(const char* attr_name, + Entry* e, + BerVarray value_array +); + +static int +get_size_func ( + Backend *be, + Connection *conn, + Operation *op, + Entry *entry, + AttributeName *attrs, + int attrsonly, + LDAPControl **ctrls +); + + +/* Two empty callback functions to avoid sending results */ +void callback_null_response( + Connection *conn, + Operation *o, + ber_tag_t tag, + ber_int_t msgid, + ber_int_t err, + const char *matched, + const char *text, + BerVarray ref, + const char *resoid, + struct berval *resdata, + struct berval *sasldata, + LDAPControl **c ) +{ +} + +void callback_null_sresult( + Connection *conn, + Operation *o, + ber_int_t err, + const char *matched, + const char *text, + BerVarray refs, + LDAPControl **c, + int nentries ) +{ +} + +struct entry_info { + int size_init; + int size_final; + int added; + Entry* entry; + struct berval* uuid; + struct timeval tv; /* time */ + enum type_of_result err; + Backend* glue_be; +}; + +int +get_entry_size( + Entry* e, + int size_init, + struct exception* result ) +{ + Attribute *a; + struct berval bv; + int i; + int tmplen; + int size=0; + + if ( result ) + result->type = SUCCESS; + + if ( e->e_dn != NULL ) { + tmplen = strlen( e->e_dn ); + size = LDIF_SIZE_NEEDED( 2, tmplen ); + } + + for ( a = e->e_attrs; a != NULL; a = a->a_next ) { + for ( i = 0; a->a_vals[i].bv_val != NULL; i++ ) { + bv = a->a_vals[i]; + tmplen = a->a_desc->ad_cname.bv_len; + size += LDIF_SIZE_NEEDED( tmplen, bv.bv_len); + } + } + if ((size < size_init) && result) { + result->type = SIZE_ERR; + } + return size; +} + + +int +merge_entry ( + Backend* be, + Connection* conn, + Entry* e, + struct berval* query_uuid, + struct exception* result ) +{ + struct entry_info info; + struct berval normdn; + struct berval prettydn; + + Operation op = {0}; + slap_callback cb = {callback_null_response, + add_func, merge_func, NULL}; + + Filter* filter = str2filter("(queryid=*)"); + + dnPrettyNormal(0, &(e->e_name), &prettydn, &normdn); + + free(e->e_name.bv_val); + e->e_name = prettydn; + e->e_nname = normdn; + + info.entry = e; + info.uuid = query_uuid; + info.size_init = 0; + info.size_final = 0; + info.added = 0; + info.glue_be = be; + info.err = SUCCESS; + cb.sc_private = &info; + + op.o_tag = LDAP_REQ_SEARCH; + op.o_protocol = LDAP_VERSION3; + op.o_ndn = conn->c_ndn; + op.o_callback = &cb; + op.o_caching_on = 1; + op.o_time = slap_get_time(); + op.o_do_not_cache = 1; + + be->be_search( be, conn, &op, NULL, &(e->e_nname), + LDAP_SCOPE_BASE, LDAP_DEREF_NEVER, 1, 0, + filter, NULL, NULL, 0 ); + result->type = info.err; + if ( result->type == SUCCESS ) + result->rc = info.added; + else + result->rc = 0; + return ( info.size_final - info.size_init ); +} + +static int +merge_func ( + Backend *be_glue, + Connection *conn, + Operation *op, + Entry *e, + AttributeName *attrs, + int attrsonly, + LDAPControl **ctrls +) +{ + Backend* be; + char *new_attr_name; + Attribute *a_new, *a; + int i=0; + int rc=0; + + int count; + struct timeval time; /* time */ + long timediff; /* time */ + slap_callback *tmp = op->o_callback; + struct entry_info* info = tmp->sc_private; + Filter* filter = str2filter("(queryid=*)"); + Entry* entry = info->entry; + struct berval* uuid = info->uuid; + Modifications *modhead = NULL; + Modifications *mod; + Modifications **modtail = &modhead; + AttributeDescription* a_new_desc; + const char *text = NULL; + + info->err = SUCCESS; + + be = select_backend(&entry->e_nname, 0, 0); + + info->size_init = get_entry_size(e, 0, 0); + a_new = entry->e_attrs; + + while (a_new != NULL) { + a_new_desc = a_new->a_desc; + mod = (Modifications *) malloc( sizeof(Modifications) ); + mod->sml_op = LDAP_MOD_REPLACE; + ber_dupbv(&(mod->sml_type), &(a_new_desc->ad_cname)); + + for (count=0; a_new->a_vals[count].bv_val; count++) + ; + mod->sml_bvalues = (struct berval*) malloc( + (count+1) * sizeof( struct berval) ); + + for (i=0; i < count; i++) { + ber_dupbv(mod->sml_bvalues+i, a_new->a_vals+i); + } + + mod->sml_bvalues[count].bv_val = 0; + mod->sml_bvalues[count].bv_len = 0; + + mod->sml_desc = NULL; + slap_bv2ad(&mod->sml_type, &mod->sml_desc, &text); + mod->sml_next =NULL; + *modtail = mod; + modtail = &mod->sml_next; + a_new = a_new->a_next; + } + + /* add query UUID to queryid attribute */ + mod = (Modifications *) ch_malloc( sizeof(Modifications) ); + mod->sml_op = LDAP_MOD_ADD; + mod->sml_desc = slap_schema.si_ad_queryid; + ber_dupbv(&(mod->sml_type), &(mod->sml_desc->ad_cname)); + mod->sml_bvalues = (BerVarray) ch_malloc( 2 * sizeof( struct berval ) ); + ber_dupbv( mod->sml_bvalues, uuid ); + mod->sml_bvalues[1].bv_val = NULL; + mod->sml_bvalues[1].bv_len = 0; + *modtail = mod; + mod->sml_next = NULL; + + if (be->be_modify(be, conn, op, &(entry->e_name), + &(entry->e_nname), modhead) != 0 ) { + info->err = MERGE_ERR; + return 0; + } + op->o_callback->sc_sendentry = get_size_func; + op->o_callback->sc_sresult = NULL; + + if (be->be_search( be, conn, op, NULL, &(entry->e_nname), + LDAP_SCOPE_BASE, LDAP_DEREF_NEVER, 1, 0, + filter, NULL, NULL, 0 ) != 0) { + info->err = GET_SIZE_ERR; + } + return 0; +} + +void +add_func ( + Connection *conn, + Operation *op, + ber_int_t err, + const char *matched, + const char *text, + BerVarray refs, + LDAPControl **ctrls, + int nentries +) +{ + slap_callback *tmp = op->o_callback; + struct entry_info *info = tmp->sc_private; + Entry* entry = info->entry; + struct berval* uuid = info->uuid; + Backend* be; + BerVarray value_array; + Entry *e; + Attribute *a; + + struct timeval time; /* time */ + long timediff; /* time */ + + /* + * new entry, construct an entry with + * the projected attributes + */ + if (nentries) + return; + + be = select_backend(&entry->e_nname, 0, 0); + e = (Entry*)malloc(sizeof(Entry)); + + ber_dupbv(&e->e_name,&entry->e_name); + ber_dupbv(&e->e_nname,&entry->e_nname); + + e->e_private = 0; + e->e_attrs = 0; + e->e_bv.bv_val = 0; + + /* add queryid attribute */ + value_array = (struct berval *)malloc(2 * sizeof( struct berval) ); + ber_dupbv(value_array, uuid); + value_array[1].bv_val = NULL; + value_array[1].bv_len = 0; + + a = add_attribute("queryid", e, value_array); + + /* append the attribute list from the fetched entry */ + a->a_next = entry->e_attrs; + entry->e_attrs = NULL; + + info->size_final = get_entry_size(e, 0, NULL); + if ( be->be_add( be, conn, op, e ) == 0 ) { + info->added = 1; + be_entry_release_w( be, conn, op, e ); + } else { + info->err = MERGE_ERR; + } +} + + +static Attribute* +add_attribute(const char* attr_name, + Entry* e, + BerVarray value_array) +{ + Attribute* new_attr, *last_attr; + const char* text; + + if (e->e_attrs == NULL) + last_attr = NULL; + else + for (last_attr = e->e_attrs; last_attr->a_next; + last_attr = last_attr->a_next) + ; + + new_attr = (Attribute*)malloc(sizeof(Attribute)); + if (last_attr) + last_attr->a_next = new_attr; + else + e->e_attrs = new_attr; + + new_attr->a_next = NULL; + new_attr->a_desc = NULL; + new_attr->a_vals = value_array; + slap_str2ad(attr_name, &(new_attr->a_desc), &text); + + return new_attr; +} + +static int +get_size_func ( + Backend *be, + Connection *conn, + Operation *op, + Entry *entry, + AttributeName *attrs, + int attrsonly, + LDAPControl **ctrls +) +{ + slap_callback *tmp = op->o_callback; + struct entry_info *info = tmp->sc_private; + struct exception result; + + result.type = info->err; + info->size_final = get_entry_size(entry, info->size_init, &result); + return 0; +} +#endif /* LDAP_CACHING */ diff --git a/servers/slapd/back-meta/cache-query.c b/servers/slapd/back-meta/cache-query.c new file mode 100644 index 0000000000..17eae727ee --- /dev/null +++ b/servers/slapd/back-meta/cache-query.c @@ -0,0 +1,462 @@ +/* Copyright (c) 2003 by International Business Machines, Inc. + * + * International Business Machines, Inc. (hereinafter called IBM) grants + * permission under its copyrights to use, copy, modify, and distribute this + * Software with or without fee, provided that the above copyright notice and + * all paragraphs of this notice appear in all copies, and that the name of IBM + * not be used in connection with the marketing of any product incorporating + * the Software or modifications thereof, without specific, written prior + * permission. + * + * THE SOFTWARE IS PROVIDED "AS IS", AND IBM DISCLAIMS ALL WARRANTIES, + * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A + * PARTICULAR PURPOSE. IN NO EVENT SHALL IBM BE LIABLE FOR ANY SPECIAL, + * DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER ARISING + * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE, EVEN + * IF IBM IS APPRISED OF THE POSSIBILITY OF SUCH DAMAGES. + */ + +#include "portable.h" + +#include + +#include "slap.h" +#include "../back-ldap/back-ldap.h" +#include "back-meta.h" + +#ifdef LDAP_CACHING +static void add_query_on_top (query_manager*, CachedQuery*); +static int base_scope_compare(struct berval* dn_stored, + struct berval* dn_incoming, int scope_stored, + int scope_incoming); + +/* check whether query is contained in any of + * the cached queries in template template_index + */ +int +query_containment(query_manager* qm, + Query* query, + int template_index) +{ + QueryTemplate* templa= qm->templates; + CachedQuery* qc; + Query* q; + Query* prev_q; + Filter* inputf = query->filter; + struct berval* base = &(query->base); + int scope = query->scope; + int i,res=0; + Filter* fs; + Filter* fi; + int ret, rc; + const char* text; + + MatchingRule* mrule = NULL; + if (inputf != NULL) { +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, "Lock QC index = %d\n", + template_index, 0, 0 ); +#else + Debug( LDAP_DEBUG_ANY, "Lock QC index = %d\n", + template_index, 0, 0 ); +#endif + ldap_pvt_thread_rdwr_rlock(&(templa[template_index].t_rwlock)); + for(qc=templa[template_index].query; qc != NULL; qc= qc->next) { + q = (Query*)qc; + if(base_scope_compare(&(q->base), base, q->scope, scope)) { + fi = inputf; + fs = q->filter; + do { + res=0; + switch (fs->f_choice) { + case LDAP_FILTER_EQUALITY: + if (fi->f_choice == LDAP_FILTER_EQUALITY) + mrule = fs->f_ava->aa_desc->ad_type->sat_equality; + else + ret = 1; + break; + case LDAP_FILTER_GE: + case LDAP_FILTER_LE: + mrule = fs->f_ava->aa_desc->ad_type->sat_ordering; + break; + default: + mrule = NULL; + } + if (mrule) { + rc = value_match(&ret, fs->f_ava->aa_desc, mrule, + SLAP_MR_ASSERTION_SYNTAX_MATCH, + &(fi->f_ava->aa_value), + &(fs->f_ava->aa_value), &text); + if (rc != LDAP_SUCCESS) { +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, + "Unlock: Exiting QC index=%d\n", + template_index, 0, 0 ); +#else + Debug( LDAP_DEBUG_ANY, + "Unlock: Exiting QC index=%d\n", + template_index, 0, 0 ); +#endif + ldap_pvt_thread_rdwr_runlock(&(templa[template_index].t_rwlock)); +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, + "query_containment: Required " + "matching rule not defined for " + "a filter attribute", + 0, 0, 0 ); +#else + Debug( LDAP_DEBUG_ANY, + "query_containment: Required " + "matching rule not defined for " + "a filter attribute", + 0, 0, 0 ); +#endif + return 0; + } + } + switch (fs->f_choice) { + case LDAP_FILTER_AND: + fs = fs->f_and; + fi = fi->f_and; + res=1; + break; + case LDAP_FILTER_SUBSTRINGS: + /* check if the equality query can be + * answered with cached substring query */ + if ((fi->f_choice == LDAP_FILTER_EQUALITY) + && substr_containment_equality( + fs, fi)) + res=1; + /* check if the substring query can be + * answered with cached substring query */ + if ((fi->f_choice ==LDAP_FILTER_SUBSTRINGS + ) && substr_containment_substr( + fs, fi)) + res= 1; + fs=fs->f_next; + fi=fi->f_next; + break; + case LDAP_FILTER_PRESENT: + res=1; + fs=fs->f_next; + fi=fi->f_next; + break; + case LDAP_FILTER_EQUALITY: + if (ret == 0) + res = 1; + fs=fs->f_next; + fi=fi->f_next; + break; + case LDAP_FILTER_GE: + if (ret >= 0) + res = 1; + fs=fs->f_next; + fi=fi->f_next; + break; + case LDAP_FILTER_LE: + if (ret <= 0) + res = 1; + fs=fs->f_next; + fi=fi->f_next; + break; + default: + break; + } + } while((res) && (fi != NULL) && (fs != NULL)); + + if(res) { + ldap_pvt_thread_mutex_lock(&qm->lru_mutex); + if (qm->lru_top != qc) { + remove_query(qm, qc); + add_query_on_top(qm, qc); + } + ldap_pvt_thread_mutex_unlock(&qm->lru_mutex); + return 1; + } + } + } +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, + "Not answerable: Unlock QC index=%d\n", + template_index, 0, 0 ); +#else + Debug( LDAP_DEBUG_ANY, + "Not answerable: Unlock QC index=%d\n", + template_index, 0, 0 ); +#endif + ldap_pvt_thread_rdwr_runlock(&(templa[template_index].t_rwlock)); + } + return 0; +} + +/* remove_query from LRU list */ + +void +remove_query (query_manager* qm, CachedQuery* qc) +{ + CachedQuery* up; + CachedQuery* down; + + if (!qc) + return; + + up = qc->lru_up; + down = qc->lru_down; + + if (!up) + qm->lru_top = down; + + if (!down) + qm->lru_bottom = up; + + if (down) + down->lru_up = up; + + if (up) + up->lru_down = down; + + qc->lru_up = qc->lru_down = NULL; +} + +/* add query on top of LRU list */ +void +add_query_on_top (query_manager* qm, CachedQuery* qc) +{ + CachedQuery* top = qm->lru_top; + Query* q = (Query*)qc; + + qm->lru_top = qc; + + if (top) + top->lru_up = qc; + else + qm->lru_bottom = qc; + + qc->lru_down = top; + qc->lru_up = NULL; +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, "Base of added query = %s\n", + q->base.bv_val, 0, 0 ); +#else + Debug( LDAP_DEBUG_ANY, "Base of added query = %s\n", + q->base.bv_val, 0, 0 ); +#endif +} + +void +free_query (CachedQuery* qc) +{ + Query* q = (Query*)qc; + free(qc->q_uuid); + filter_free(q->filter); + free (q->base.bv_val); + + free(q->attrs); + free(qc); +} + +/* compare base and scope of incoming and cached queries */ +int base_scope_compare( + struct berval* dn_stored, + struct berval* dn_incoming, + int scope_stored, + int scope_incoming ) +{ + struct berval* ndn_incoming = NULL; + struct berval* pdn_incoming = NULL; + struct berval* ndn_stored = NULL; + + int i; + + if (scope_stored < scope_incoming) + return 0; + + dnNormalize(NULL, dn_incoming, &ndn_incoming); + dnNormalize(NULL, dn_stored, &ndn_stored); + + i = dnIsSuffix(ndn_incoming, ndn_stored); + + if ( i == 0 ) + return 0; + + switch(scope_stored) { + case LDAP_SCOPE_BASE: + if (strlen(ndn_incoming->bv_val) == strlen(ndn_stored->bv_val)) + return 1; + else + return 0; + break; + case LDAP_SCOPE_ONELEVEL: + switch(scope_incoming){ + case LDAP_SCOPE_BASE: + dnParent(ndn_incoming, pdn_incoming); + if(strcmp(pdn_incoming->bv_val,ndn_stored->bv_val) == 0) + return 1; + else + return 0; + break; + case LDAP_SCOPE_ONELEVEL: + if (ndn_incoming->bv_len == ndn_stored->bv_len) + return 1; + else + return 0; + break; + default: + return 0; + break; + } + case LDAP_SCOPE_SUBTREE: + return 1; + break; + default: + return 0; + break; + } +} + +/* Add query to query cache */ +void add_query( + query_manager* qm, + Query* query, + int template_index, + char* uuid, + struct exception* result) +{ + CachedQuery* new_cached_query = (CachedQuery*) malloc(sizeof(CachedQuery)); + QueryTemplate* templ = (qm->templates)+template_index; + Query* new_query; + new_cached_query->template_id = template_index; + new_cached_query->q_uuid = uuid; + new_cached_query->lru_up = NULL; + new_cached_query->lru_down = NULL; + new_cached_query->expiry_time = slap_get_time() + templ->ttl; +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, "Added query expires at %d\n", + new_cached_query->expiry_time, 0, 0 ); +#else + Debug( LDAP_DEBUG_ANY, "Added query expires at %d\n", + new_cached_query->expiry_time, 0, 0 ); +#endif + new_query = (Query*)new_cached_query; + + new_query->base.bv_val = ch_strdup(query->base.bv_val); + new_query->base.bv_len = query->base.bv_len; + new_query->scope = query->scope; + new_query->filter = query->filter; + new_query->attrs = query->attrs; + + /* Adding a query */ +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, "Lock AQ index = %d\n", + template_index, 0, 0 ); +#else + Debug( LDAP_DEBUG_ANY, "Lock AQ index = %d\n", + template_index, 0, 0 ); +#endif + ldap_pvt_thread_rdwr_wlock(&templ->t_rwlock); + if (templ->query == NULL) + templ->query_last = new_cached_query; + else + templ->query->prev = new_cached_query; + new_cached_query->next = templ->query; + new_cached_query->prev = NULL; + templ->query = new_cached_query; + templ->no_of_queries++; +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, "TEMPLATE %d QUERIES++ %d\n", + template_index, templ->no_of_queries, 0 ); +#else + Debug( LDAP_DEBUG_ANY, "TEMPLATE %d QUERIES++ %d\n", + template_index, templ->no_of_queries, 0 ); +#endif + +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, "Unlock AQ index = %d \n", + template_index, 0, 0 ); +#else + Debug( LDAP_DEBUG_ANY, "Unlock AQ index = %d \n", + template_index, 0, 0 ); +#endif + ldap_pvt_thread_rdwr_wunlock(&templ->t_rwlock); + + /* Adding on top of LRU list */ + ldap_pvt_thread_mutex_lock(&qm->lru_mutex); + add_query_on_top(qm, new_cached_query); + ldap_pvt_thread_mutex_unlock(&qm->lru_mutex); + +} + +/* remove bottom query of LRU list from the query cache */ +char* cache_replacement(query_manager* qm) +{ + char* result = (char*)(malloc(40)); + CachedQuery* bottom; + QueryTemplate* templ; + CachedQuery* query_curr; + int temp_id; + + ldap_pvt_thread_mutex_lock(&qm->lru_mutex); + bottom = qm->lru_bottom; + + if (!bottom) { +#ifdef NEW_LOGGING + LDAP_LOG ( BACK_META, DETAIL1, + "Cache replacement invoked without " + "any query in LRU list\n", 0, 0, 0 ); +#else + Debug ( LDAP_DEBUG_ANY, + "Cache replacement invoked without " + "any query in LRU list\n", 0, 0, 0 ); +#endif + return 0; + } + + temp_id = bottom->template_id; + remove_query(qm, bottom); + ldap_pvt_thread_mutex_unlock(&qm->lru_mutex); + + strcpy(result, bottom->q_uuid); + +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, "Lock CR index = %d\n", temp_id, 0, 0 ); +#else + Debug( LDAP_DEBUG_ANY, "Lock CR index = %d\n", temp_id, 0, 0 ); +#endif + ldap_pvt_thread_rdwr_wlock(&(qm->templates[temp_id].t_rwlock)); + remove_from_template(bottom, (qm->templates+temp_id)); +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, "TEMPLATE %d QUERIES-- %d\n", + temp_id, qm->templates[temp_id].no_of_queries, 0 ); +#else + Debug( LDAP_DEBUG_ANY, "TEMPLATE %d QUERIES-- %d\n", + temp_id, qm->templates[temp_id].no_of_queries, 0 ); +#endif +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, "Unlock CR index = %d\n", temp_id, 0, 0 ); +#else + Debug( LDAP_DEBUG_ANY, "Unlock CR index = %d\n", temp_id, 0, 0 ); +#endif + ldap_pvt_thread_rdwr_wunlock(&(qm->templates[temp_id].t_rwlock)); + free_query(bottom); + return result; +} + +void +remove_from_template (CachedQuery* qc, QueryTemplate* template) +{ + if (!qc->prev && !qc->next) { + template->query_last = template->query = NULL; + } else if (qc->prev == NULL) { + qc->next->prev = NULL; + template->query = qc->next; + } else if (qc->next == NULL) { + qc->prev->next = NULL; + template->query_last = qc->prev; + } else { + qc->next->prev = qc->prev; + qc->prev->next = qc->next; + } + + template->no_of_queries--; +} +#endif diff --git a/servers/slapd/back-meta/cache-remove.c b/servers/slapd/back-meta/cache-remove.c new file mode 100644 index 0000000000..25c97dc563 --- /dev/null +++ b/servers/slapd/back-meta/cache-remove.c @@ -0,0 +1,183 @@ +/* Copyright (c) 2003 by International Business Machines, Inc. + * + * International Business Machines, Inc. (hereinafter called IBM) grants + * permission under its copyrights to use, copy, modify, and distribute this + * Software with or without fee, provided that the above copyright notice and + * all paragraphs of this notice appear in all copies, and that the name of IBM + * not be used in connection with the marketing of any product incorporating + * the Software or modifications thereof, without specific, written prior + * permission. + * + * THE SOFTWARE IS PROVIDED "AS IS", AND IBM DISCLAIMS ALL WARRANTIES, + * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A + * PARTICULAR PURPOSE. IN NO EVENT SHALL IBM BE LIABLE FOR ANY SPECIAL, + * DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER ARISING + * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE, EVEN + * IF IBM IS APPRISED OF THE POSSIBILITY OF SUCH DAMAGES. + */ + +#include "portable.h" + +#include + +#include +#include +#include + +#include "slap.h" +#include "ldif.h" +#include "../back-ldap/back-ldap.h" +#include "back-meta.h" +#include "ldap_pvt.h" +#undef ldap_debug /* silence a warning in ldap-int.h */ +#include "ldap_log.h" +#include "../../../libraries/libldap/ldap-int.h" +#include + +#ifdef LDAP_CACHING +static int +remove_func ( + Backend *be, + Connection *conn, + Operation *op, + Entry *entry, + AttributeName *attrs, + int attrsonly, + LDAPControl **ctrls +); + +struct query_info { + int freed; + int deleted; + struct berval* uuid; + struct timeval tv; + enum type_of_result err; +}; + +int +remove_query_data ( + Backend* be, + Connection* conn, + struct berval* query_uuid, + struct exception* result) +{ + struct query_info info; + char filter_str[64]; + Operation op = {0}; + Filter* filter; + struct timeval time_in; + struct timeval time_out; + long timediff; + + slap_callback cb = {callback_null_response, + callback_null_sresult, remove_func, NULL}; + sprintf(filter_str, "(queryid=%s)", query_uuid->bv_val); + filter = str2filter(filter_str); + info.uuid = query_uuid; + info.freed = 0; + info.deleted = 0; + info.err = SUCCESS; + cb.sc_private = &info; + + op.o_tag = LDAP_REQ_SEARCH; + op.o_protocol = LDAP_VERSION3; + op.o_ndn = conn->c_ndn; + op.o_callback = &cb; + op.o_time = slap_get_time(); + op.o_do_not_cache = 1; + op.o_caching_on = 1; + be->be_search( be, conn, &op, NULL, &(be->be_nsuffix[0]), + LDAP_SCOPE_SUBTREE, LDAP_DEREF_NEVER, 0, 0, + filter, NULL, NULL, 0 ); + result->type = info.err; + result->rc = info.deleted; + return info.freed; +} + +static int +remove_func ( + Backend *be, + Connection *conn, + Operation *op, + Entry *entry, + AttributeName *attrs, + int attrsonly, + LDAPControl **ctrls +) +{ + slap_callback *tmp = op->o_callback; + struct query_info* info = tmp->sc_private; +#if 0 /* ??? pdn is not used anywhere */ + struct berval pdn; +#endif + int count = 0; + int size; + struct timeval time_in; + struct timeval time_out; + long timediff; + Modifications* mod; + + Attribute* attr; + size = get_entry_size(entry, 0, NULL); + + for (attr = entry->e_attrs; attr!= NULL; attr = attr->a_next) { + if (attr->a_desc == slap_schema.si_ad_queryid) { + for (count=0; attr->a_vals[count].bv_val; count++) + ; + break; + } + } + + if (count == 0) { + info->err = REMOVE_ERR; + return 0; + } + if (count == 1) { +#if 0 /* ??? pdn is not used anywhere */ + dnPretty2(NULL, &entry->e_nname, &pdn); +#endif + info->freed += size; +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, + "DELETING ENTRY SIZE=%d TEMPLATE=%s\n", + size, attr->a_vals[0].bv_val, 0 ); +#else + Debug( LDAP_DEBUG_ANY, "DELETING ENTRY SIZE=%d TEMPLATE=%s\n", + size, attr->a_vals[0].bv_val, 0 ); +#endif + + if (be->be_delete (be, conn, op, &entry->e_name, &entry->e_nname)) { + info->err = REMOVE_ERR; + } else { + info->deleted++; + } + return 0; + } + + mod = (Modifications*)malloc(sizeof(Modifications)); + mod->sml_op = LDAP_MOD_DELETE; + mod->sml_type.bv_len = sizeof("queryid"); + mod->sml_type.bv_val = "queryid"; + mod->sml_desc = slap_schema.si_ad_queryid; + mod->sml_bvalues = (struct berval*) malloc( 2 * sizeof( struct berval) ); + ber_dupbv(mod->sml_bvalues, info->uuid); + mod->sml_bvalues[1].bv_val = NULL; + mod->sml_bvalues[1].bv_len = 0; + mod->sml_next = NULL; +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, + "REMOVING TEMP ATTR : TEMPLATE=%s\n", + attr->a_vals[0].bv_val, 0, 0 ); +#else + Debug( LDAP_DEBUG_ANY, "REMOVING TEMP ATTR : TEMPLATE=%s\n", + attr->a_vals[0].bv_val, 0, 0 ); +#endif + if (be->be_modify(be, conn, op, &(entry->e_name), &(entry->e_nname), mod)) { + info->err = REMOVE_ERR; + } + info->freed += LDIF_SIZE_NEEDED(9, (strlen(info->uuid->bv_val))); + + return 0; +} + +#endif /* LDAP_CACHING */ diff --git a/servers/slapd/back-meta/cache-search.c b/servers/slapd/back-meta/cache-search.c new file mode 100644 index 0000000000..c6e7127317 --- /dev/null +++ b/servers/slapd/back-meta/cache-search.c @@ -0,0 +1,1724 @@ +/* + * Copyright (c) 2003 by International Business Machines, Inc. + * + * International Business Machines, Inc. (hereinafter called IBM) grants + * permission under its copyrights to use, copy, modify, and distribute this + * Software with or without fee, provided that the above copyright notice and + * all paragraphs of this notice appear in all copies, and that the name of IBM + * not be used in connection with the marketing of any product incorporating + * the Software or modifications thereof, without specific, written prior + * permission. + * + * THE SOFTWARE IS PROVIDED "AS IS", AND IBM DISCLAIMS ALL WARRANTIES, + * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A + * PARTICULAR PURPOSE. IN NO EVENT SHALL IBM BE LIABLE FOR ANY SPECIAL, + * DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER ARISING + * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE, EVEN + * IF IBM IS APPRISED OF THE POSSIBILITY OF SUCH DAMAGES. + * + * + * This software is based on the backends back-ldap and back-meta, implemented + * by Howard Chu , Mark Valence + * , Pierangelo Masarati and other + * contributors. + * + * The original copyright statements follow. + * Copyright 1998-2002 The OpenLDAP Foundation, All Rights Reserved. + * COPYING RESTRICTIONS APPLY, see COPYRIGHT file + * + * Copyright 2001, Pierangelo Masarati, All rights reserved. + * + * This work has been developed to fulfill the requirements + * of SysNet s.n.c. and it has been donated + * to the OpenLDAP Foundation in the hope that it may be useful + * to the Open Source community, but WITHOUT ANY WARRANTY. + * + * Permission is granted to anyone to use this software for any purpose + * on any computer system, and to alter it and redistribute it, subject + * to the following restrictions: + * + * 1. The author and SysNet s.n.c. are not responsible for the consequences + * of use of this software, no matter how awful, even if they arise from + * flaws in it. + * + * 2. The origin of this software must not be misrepresented, either by + * explicit claim or by omission. Since few users ever read sources, + * credits should appear in the documentation. + * + * 3. Altered versions must be plainly marked as such, and must not be + * misrepresented as being the original software. Since few users + * ever read sources, credits should appear in the documentation. + * SysNet s.n.c. cannot be responsible for the consequences of the + * alterations. + * + * 4. This notice may not be removed or altered. + * + * + * This software is based on the backend back-ldap, implemented + * by Howard Chu , and modified by Mark Valence + * , Pierangelo Masarati and other + * contributors. The contribution of the original software to the present + * implementation is acknowledged in this copyright statement. + * + * A special acknowledgement goes to Howard for the overall architecture + * (and for borrowing large pieces of code), and to Mark, who implemented + * from scratch the attribute/objectclass mapping. + * + * The original copyright statement follows. + * + * Copyright 1999, Howard Chu, All rights reserved. + * + * Permission is granted to anyone to use this software for any purpose + * on any computer system, and to alter it and redistribute it, subject + * to the following restrictions: + * + * 1. The author is not responsible for the consequences of use of this + * software, no matter how awful, even if they arise from flaws in it. + * + * 2. The origin of this software must not be misrepresented, either by + * explicit claim or by omission. Since few users ever read sources, + * credits should appear in the documentation. + * + * 3. Altered versions must be plainly marked as such, and must not be + * misrepresented as being the original software. Since few users + * ever read sources, credits should appear in the + * documentation. + * + * 4. This notice may not be removed or altered. + * + */ +#include "portable.h" + +#include + +#include +#include +#include + +#include "lutil.h" +#include "slap.h" +#include "../back-ldap/back-ldap.h" +#include "back-meta.h" +#include "ldap_pvt.h" +#undef ldap_debug /* silence a warning in ldap-int.h */ +#include "ldap_log.h" +#include "../../../libraries/libldap/ldap-int.h" +#include + +#ifdef LDAP_CACHING +static Entry* +meta_create_entry( + Backend *be, + struct metaconn *lc, + int target, + LDAPMessage *e, + struct exception* result +); + +static int +is_one_level_rdn( + const char *rdn, + int from +); + +static struct metaconn* +metaConnect( + struct metainfo* li, + Connection* conn, + Operation* op, + int op_type, + struct berval* nbase, + struct exception* result +); + +static void +add_filter_attrs( + AttributeName** newattrs, + AttributeName* attrs, + AttributeName* filter_attrs +); + +static int +handleLdapResult( + struct metaconn* lc, + Operation* op, + int* msgid, Backend* be, + AttributeName* attrs, + int attrsonly, + int candidates, + int cacheable, + Entry*** entry_array, + int curr_limit, + int slimit, + struct exception* result +); + +static Entry* +get_result_entry( + Backend* be, + struct metaconn* lc, + struct metasingleconn* lsc, + int* msgid, + int i, + struct timeval* tv, + struct exception* result +); + +static void +rewriteSession( + struct rewrite_info* info, + const char* rewriteContext, + const char* string, + const void* cookie, + char** base, + struct exception* result +); + +static int +get_attr_set( + AttributeName* attrs, + query_manager* qm, + int num +); + +static int +attrscmp( + AttributeName* attrs_in, + AttributeName* attrs +); + +static char* +cache_entries( + Entry** entry_array, + cache_manager* cm, + Backend* be, + Connection* conn, + struct exception* result +); + +static int +is_temp_answerable( + int attr_set, + struct berval* tempstr, + query_manager* qm, + int template_id +); + +static void +consistency_check( + Backend* be, + Backend* glue_be, + Connection* conn +); + +static int +cache_back_sentry( + Backend* be, + Connection* conn, + Operation* op, + Entry* e, + AttributeName* attrs, + int attrsonly, + LDAPControl** ctrls +); + +int +meta_back_cache_search( + Backend *be, + Connection *conn, + Operation *op, + struct berval *base, + struct berval *nbase, + int scope, + int deref, + int slimit, + int tlimit, + Filter *filt, + struct berval *filterstr, + AttributeName *attributes, + int attrsonly +) +{ + struct metainfo *li = ( struct metainfo * )be->be_private; + struct metaconn *lc; + struct metasingleconn *lsc; + cache_manager* cm = li->cm; + query_manager* qm = cm->qm; + + time_t curr_time; + + int count, rc = 0, *msgid = NULL; + char *mbase = NULL; + char *cbase = NULL; + char *uuid; + + int i = -1, last = 0, candidates = 0, op_type; + + struct berval mfilter; + struct berval *cachebase = NULL; + struct berval *ncachebase = NULL; + struct berval cache_suffix; + struct berval tempstr = {0, 0}; + + AttributeName *filter_attrs = NULL; + AttributeName *new_attrs = NULL; + AttributeName *attrs = NULL; + + Entry *e; + Entry **entry_array = NULL; + + Query query; + + int attr_set = -1; + int template_id = -1; + int answerable = 0; + int cacheable = 0; + int num_entries = 0; + int curr_limit; + int fattr_cnt=0; + + + struct exception result[1]; + + Filter* filter = str2filter(filterstr->bv_val); + slap_callback cb = {NULL, NULL, cache_back_sentry, NULL}; + + cb.sc_private = be; + + if (attributes) { + for ( count=0; attributes[ count ].an_name.bv_val; count++ ) + ; + attrs = (AttributeName*)malloc( ( count + 1 ) * + sizeof(AttributeName)); + for ( count=0; attributes[ count ].an_name.bv_val; count++ ) { + ber_dupbv(&attrs[ count ].an_name, + &attributes[ count ].an_name); + attrs[count].an_desc = attributes[count].an_desc; + } + attrs[ count ].an_name.bv_val = NULL; + attrs[ count ].an_name.bv_len = 0; + } + + result->type = SUCCESS; + result->rc = 0; + ldap_pvt_thread_mutex_lock(&cm->cache_mutex); + cm->threads++; +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, "Threads++ = %d\n", cm->threads, 0, 0 ); +#else /* !NEW_LOGGING */ + Debug( LDAP_DEBUG_ANY, "Threads++ = %d\n", cm->threads, 0, 0 ); +#endif /* !NEW_LOGGING */ + ldap_pvt_thread_mutex_unlock(&cm->cache_mutex); + + filter2template(filter, &tempstr, &filter_attrs, &fattr_cnt, result); + if (result->type != SUCCESS) + goto Catch; + +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, "query template of incoming query = %s\n", + tempstr.bv_val, 0, 0 ); +#else /* !NEW_LOGGING */ + Debug( LDAP_DEBUG_ANY, "query template of incoming query = %s\n", + tempstr.bv_val, 0, 0 ); +#endif /* !NEW_LOGGING */ + curr_limit = cm->num_entries_limit ; + + /* find attr set */ + attr_set = get_attr_set(attrs, qm, cm->numattrsets); + + query.filter = filter; + query.attrs = attrs; + query.base = *base; + query.scope = scope; + + /* check for query containment */ + if (attr_set > -1) { + for (i=0; inumtemplates; i++) { + /* find if template i can potentially answer tempstr */ + if (!is_temp_answerable(attr_set, &tempstr, qm, i)) + continue; + if (attr_set == qm->templates[i].attr_set_index) { + cacheable = 1; + template_id = i; + } +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL2, + "Entering QC, querystr = %s\n", + filterstr->bv_val, 0, 0 ); +#else /* !NEW_LOGGING */ + Debug( LDAP_DEBUG_NONE, "Entering QC, querystr = %s\n", + filterstr->bv_val, 0, 0 ); +#endif /* !NEW_LOGGING */ + answerable = (*(qm->qcfunc))(qm, &query, i); + + if (answerable) + break; + } + } + + if (answerable) { +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, "QUERY ANSWERABLE\n", 0, 0, 0 ); +#else /* !NEW_LOGGING */ + Debug( LDAP_DEBUG_ANY, "QUERY ANSWERABLE\n", 0, 0, 0 ); +#endif /* !NEW_LOGGING */ + rewriteSession(li->rwinfo, "cacheBase", nbase->bv_val, + conn, &cbase, result); + if (result->type != SUCCESS) { + ldap_pvt_thread_rdwr_runlock(&qm->templates[i].t_rwlock); + goto Catch; + } + cachebase = ber_str2bv(cbase, strlen(cbase), 0, NULL); + dnNormalize(NULL, cachebase, &ncachebase); + + op->o_caching_on = 1; + op->o_callback = &cb; + li->glue_be->be_search(li->glue_be, conn, op, cachebase, + ncachebase, scope, deref, slimit, tlimit, + filter, filterstr, attrs, attrsonly); + + ldap_pvt_thread_rdwr_runlock(&qm->templates[i].t_rwlock); + } else { +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, "QUERY NOT ANSWERABLE\n", + 0, 0, 0 ); +#else /* !NEW_LOGGING */ + Debug( LDAP_DEBUG_ANY, "QUERY NOT ANSWERABLE\n", 0, 0, 0 ); +#endif /* !NEW_LOGGING */ + + if ( scope == LDAP_SCOPE_BASE ) { + op_type = META_OP_REQUIRE_SINGLE; + } else { + op_type = META_OP_ALLOW_MULTIPLE; + } + + lc = metaConnect(li, conn, op, op_type, nbase, result); + + if (result->type != SUCCESS) + goto Catch; + + ldap_pvt_thread_mutex_lock(&cm->cache_mutex); + if (cm->num_cached_queries >= cm->max_queries) { + cacheable = 0; + } + ldap_pvt_thread_mutex_unlock(&cm->cache_mutex); + + if (cacheable) { +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, + "QUERY TEMPLATE CACHEABLE\n", + 0, 0, 0); +#else /* !NEW_LOGGING */ + Debug( LDAP_DEBUG_ANY, "QUERY TEMPLATE CACHEABLE\n", + 0, 0, 0); +#endif /* !NEW_LOGGING */ + add_filter_attrs(&new_attrs, attrs, filter_attrs); + } else { + new_attrs = attrs; + } + + free(filter_attrs); + + /* + * Array of message id of each target + */ + msgid = ch_calloc( sizeof( int ), li->ntargets ); + if ( msgid == NULL ) { + result->type = CONN_ERR; + goto Catch; + } + + /* + if (slimit > 0 && (slimit <= cm->num_entries_limit)) + slimit = cm->num_entries_limit; + */ + + /* + * Inits searches + */ + + for ( i = 0, lsc = lc->conns; !META_LAST(lsc); ++i, ++lsc ) { + char *realbase = ( char * )base->bv_val; + int realscope = scope; + ber_len_t suffixlen; + char *mapped_filter, **mapped_attrs; + + /* FIXME: Check for more than one targets */ + if ( meta_back_is_candidate( + &li->targets[i]->suffix, nbase )) + lsc->candidate = META_CANDIDATE; + + if ( lsc->candidate != META_CANDIDATE ) + continue; + + if ( deref != -1 ) { + ldap_set_option( lsc->ld, LDAP_OPT_DEREF, + ( void * )&deref); + } + if ( tlimit != -1 ) { + ldap_set_option( lsc->ld, LDAP_OPT_TIMELIMIT, + ( void * )&tlimit); + } + if ( slimit != -1 ) { + ldap_set_option( lsc->ld, LDAP_OPT_SIZELIMIT, + ( void * )&slimit); + } + + /* + * modifies the base according to the scope, if required + */ + suffixlen = li->targets[ i ]->suffix.bv_len; + if ( suffixlen > nbase->bv_len ) { + switch ( scope ) { + case LDAP_SCOPE_SUBTREE: + /* + * make the target suffix the new base + * FIXME: this is very forgiving, + * because illegal bases may be turned + * into the suffix of the target. + */ + if ( dnIsSuffix( + &li->targets[ i ]->suffix, + nbase ) ) { + realbase = + li->targets[i]->suffix.bv_val; + } else { + /* + * this target is no longer + * candidate + */ + lsc->candidate = + META_NOT_CANDIDATE; + continue; + } + break; + + case LDAP_SCOPE_ONELEVEL: + if ( is_one_level_rdn( + li->targets[ i ]->suffix.bv_val, + suffixlen - nbase->bv_len - 1 ) + && dnIsSuffix( + &li->targets[ i ]->suffix, + nbase ) ) { + /* + * if there is exactly one + * level, make the target suffix + * the new base, and make scope + * "base" + */ + realbase = + li->targets[i]->suffix.bv_val; + realscope = LDAP_SCOPE_BASE; + break; + } /* else continue with the next case */ + case LDAP_SCOPE_BASE: + /* + * this target is no longer candidate + */ + lsc->candidate = META_NOT_CANDIDATE; + continue; + } + } + + /* + * Rewrite the search base, if required + */ + + rewriteSession(li->targets[i]->rwinfo, "searchBase", + realbase, conn, &mbase, result); + + if (result->type != SUCCESS) + goto Catch; + + if ( mbase == NULL ) { + mbase = realbase; + } + + /* + * Rewrite the search filter, if required + */ + rewriteSession( li->targets[i]->rwinfo, "searchFilter", + filterstr->bv_val, conn, + &mfilter.bv_val, result); + if (result->type != SUCCESS) + goto Catch; + + if ( mfilter.bv_val != NULL && mfilter.bv_val[ 0 ] + != '\0') { + mfilter.bv_len = strlen( mfilter.bv_val ); + } else { + if ( mfilter.bv_val != NULL ) { + free( mfilter.bv_val ); + } + mfilter = *filterstr; + } + + /* + * Maps attributes in filter + */ + mapped_filter = ldap_back_map_filter( + &li->targets[i]->at_map, + &li->targets[i]->oc_map, &mfilter, 0 ); + if ( mapped_filter == NULL ) { + mapped_filter = ( char * )mfilter.bv_val; + } else { + if ( mfilter.bv_val != filterstr->bv_val ) { + free( mfilter.bv_val ); + } + } + mfilter.bv_val = NULL; + mfilter.bv_len = 0; + + /* + * Maps required attributes + */ + mapped_attrs = ldap_back_map_attrs( + &li->targets[ i ]->at_map, + new_attrs, 0 ); + if ( mapped_attrs == NULL && new_attrs) { + for ( count=0; + new_attrs[ count ].an_name.bv_val; + count++) + ; + mapped_attrs = ch_malloc( ( count + 1 ) * + sizeof(char *)); + for ( count=0; + new_attrs[ count ].an_name.bv_val; + count++ ) { + mapped_attrs[ count ] = + new_attrs[count].an_name.bv_val; + } + mapped_attrs[ count ] = NULL; + } + + /* + * Starts the search + */ + msgid[ i ] = ldap_search( lsc->ld, mbase, realscope, + mapped_filter, mapped_attrs, + attrsonly ); + if ( msgid[ i ] == -1 ) { + lsc->candidate = META_NOT_CANDIDATE; + continue; + } + + if ( mapped_attrs ) { + free( mapped_attrs ); + mapped_attrs = NULL; + } + + if ( mapped_filter != filterstr->bv_val ) { + free( mapped_filter ); + mapped_filter = NULL; + } + + if ( mbase != realbase ) { + free( mbase ); + mbase = NULL; + } + + ++candidates; + } + + num_entries = handleLdapResult(lc, op, msgid, be, attrs, + attrsonly, candidates, cacheable, &entry_array, + curr_limit, slimit, result); + + if (result->type != SUCCESS) + goto Catch; + if (cacheable && (num_entries <= curr_limit)) { +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, + "QUERY CACHEABLE\n", 0, 0, 0 ); +#else /* !NEW_LOGGING */ + Debug( LDAP_DEBUG_ANY, "QUERY CACHEABLE\n", 0, 0, 0 ); +#endif /* !NEW_LOGGING */ + uuid = cache_entries(entry_array, cm, li->glue_be, + conn, result); +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, + "Added query %s UUID %s ENTRIES %d\n", + filterstr->bv_val, uuid, num_entries ); +#else /* !NEW_LOGGING */ + Debug( LDAP_DEBUG_ANY, + "Added query %s UUID %s ENTRIES %d\n", + filterstr->bv_val, uuid, num_entries ); +#endif /* !NEW_LOGGING */ + + if (result->type != SUCCESS) + goto Catch; + (*(qm->addfunc))(qm, &query, template_id, uuid, result); + if (result->type != SUCCESS) + goto Catch; + filter = 0; + attrs = 0; + } + } + +Catch: + switch (result->type) { + case SUCCESS: + rc=0; + break; + case FILTER_ERR: +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, + "Invalid template error\n", 0, 0, 0 ); +#else /* !NEW_LOGGING */ + Debug( LDAP_DEBUG_ANY, "Invalid template error\n", + 0, 0, 0 ); +#endif /* !NEW_LOGGING */ + break; + case CONN_ERR: + rc = -1; +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, + "Could not connect to a remote server\n", + 0, 0, 0 ); +#else /* !NEW_LOGGING */ + Debug( LDAP_DEBUG_ANY, + "Could not connect to a remote server\n", + 0, 0, 0 ); +#endif /* !NEW_LOGGING */ + send_ldap_result(conn, op, LDAP_OTHER, + NULL, "Connection error", + NULL, NULL ); + break; + case RESULT_ERR: + rc = -1; +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, + "Error in handling ldap_result\n", 0, 0, 0 ); +#else /* !NEW_LOGGING */ + Debug( LDAP_DEBUG_ANY, + "Error in handling ldap_result\n", 0, 0, 0 ); +#endif /* !NEW_LOGGING */ + break; + case REWRITING_ERR: + rc = -1; + if (result->rc == REWRITE_REGEXEC_UNWILLING) { + send_ldap_result( conn, op, + LDAP_UNWILLING_TO_PERFORM, + NULL, "Unwilling to perform", + NULL, NULL ); + } else { + send_ldap_result( conn, op, LDAP_OTHER, + NULL, "Rewrite error", + NULL, NULL ); + } + break; + case MERGE_ERR: + rc = -1; +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, + "Error in merging entry \n", 0, 0, 0 ); +#else /* !NEW_LOGGING */ + Debug( LDAP_DEBUG_ANY, + "Error in merging entry \n", 0, 0, 0 ); +#endif /* !NEW_LOGGING */ + break; + case REMOVE_ERR: + rc = -1; +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, + "Error in removing query \n", + 0, 0, 0 ); +#else /* !NEW_LOGGING */ + Debug( LDAP_DEBUG_ANY, "Error in removing query \n", + 0, 0, 0 ); +#endif /* !NEW_LOGGING */ + break; + default: + /* assert(0); */ + } + + ldap_pvt_thread_mutex_lock(&cm->consistency_mutex); + curr_time = slap_get_time(); + if (curr_time - cm->consistency_time > cm->consistency_cycle_time) { + cm->consistency_time = curr_time; + consistency_check(be, li->glue_be, conn); + } + ldap_pvt_thread_mutex_unlock(&cm->consistency_mutex); + + if ( msgid ) { + ch_free( msgid ); + } + if (entry_array) { + for (i=0; (e = entry_array[i]); i++) { + entry_free(e); + } + free(entry_array); + } + if (filter) + filter_free(filter); + + if (new_attrs) { + if (new_attrs != attrs) + free(new_attrs); + } + + if (attrs) + free(attrs); + + if (tempstr.bv_val ) { + free(tempstr.bv_val); + } + ldap_pvt_thread_mutex_lock(&cm->cache_mutex); + cm->threads--; +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, "Threads-- = %d\n", cm->threads, 0, 0 ); +#else /* !NEW_LOGGING */ + Debug( LDAP_DEBUG_ANY, "Threads-- = %d\n", cm->threads, 0, 0 ); +#endif /* !NEW_LOGGING */ + ldap_pvt_thread_mutex_unlock(&cm->cache_mutex); + return rc; +} + + +static Entry* +meta_create_entry ( + Backend *be, + struct metaconn *lc, + int target, + LDAPMessage *e, + struct exception* result +) +{ + struct metainfo *li = ( struct metainfo * )be->be_private; + struct berval a, mapped; + Entry* ent; + BerElement ber = *e->lm_ber; + Attribute *attr, **attrp; + struct berval dummy = { 0, NULL }; + struct berval *bv, bdn; + const char *text; + char* ename = NULL; + if ( ber_scanf( &ber, "{m{", &bdn ) == LBER_ERROR ) { + result->type = CREATE_ENTRY_ERR; + return NULL; + } + ent = (Entry*)malloc(sizeof(Entry)); + + /* + * Rewrite the dn of the result, if needed + */ + rewriteSession( li->targets[ target ]->rwinfo, "searchResult", + bdn.bv_val, lc->conn, &ent->e_name.bv_val, result ); + + if (result->type != SUCCESS) { + return NULL; + } + if ( ent->e_name.bv_val == NULL ) { + ber_dupbv(&(ent->e_name), &bdn); + } else { +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, + "[rw] searchResult[%d]: \"%s\" -> \"%s\"\n", + target, bdn.bv_val, ent->e_name.bv_val ); +#else /* !NEW_LOGGING */ + Debug( LDAP_DEBUG_ARGS, "rw> searchResult[%d]: \"%s\"" + " -> \"%s\"\n", + target, bdn.bv_val, ent->e_name.bv_val ); +#endif /* !NEW_LOGGING */ + ent->e_name.bv_len = strlen( ent->e_name.bv_val ); + } + + /* + * Note: this may fail if the target host(s) schema differs + * from the one known to the meta, and a DN with unknown + * attributes is returned. + * + * FIXME: should we log anything, or delegate to dnNormalize2? + */ + dnNormalize2( NULL, &ent->e_name, &ent->e_nname ); + + /* + if ( dnNormalize2( NULL, &ent->e_name, &ent->e_nname ) != LDAP_SUCCESS ) { + return LDAP_INVALID_DN_SYNTAX; + } + */ + + /* + * cache dn + */ + if ( li->cache.ttl != META_DNCACHE_DISABLED ) { + meta_dncache_update_entry( &li->cache, &ent->e_nname, target ); + } + + ent->e_id = 0; + ent->e_attrs = 0; + ent->e_private = 0; + ent->e_bv.bv_val = 0; + + attrp = &ent->e_attrs; + + while ( ber_scanf( &ber, "{m", &a ) != LBER_ERROR ) { + ldap_back_map( &li->targets[ target ]->at_map, &a, &mapped, 1 ); + if ( mapped.bv_val == NULL ) { + continue; + } + attr = ( Attribute * )ch_malloc( sizeof( Attribute ) ); + if ( attr == NULL ) { + continue; + } + attr->a_flags = 0; + attr->a_next = 0; + attr->a_desc = NULL; + if ( slap_bv2ad( &mapped, &attr->a_desc, &text ) != LDAP_SUCCESS) { + if ( slap_bv2undef_ad( &mapped, &attr->a_desc, &text ) + != LDAP_SUCCESS) { +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, + "slap_bv2undef_ad(%s): %s\n", + mapped.bv_val, text, 0 ); +#else /* !NEW_LOGGING */ + Debug( LDAP_DEBUG_ANY, + "slap_bv2undef_ad(%s): " + "%s\n%s", mapped.bv_val, text, "" ); +#endif /* !NEW_LOGGING */ + ch_free( attr ); + continue; + } + } + + /* no subschemaSubentry */ + if ( attr->a_desc == slap_schema.si_ad_subschemaSubentry ) { + ch_free(attr); + continue; + } + + if ( ber_scanf( &ber, "[W]", &attr->a_vals ) == LBER_ERROR + || attr->a_vals == NULL ) { + attr->a_vals = &dummy; + } else if ( attr->a_desc == slap_schema.si_ad_objectClass || + attr->a_desc == + slap_schema.si_ad_structuralObjectClass) { + int i, last; + for ( last = 0; attr->a_vals[ last ].bv_val; ++last ) + ; + for ( i = 0, bv = attr->a_vals; bv->bv_val; bv++,i++ ) { + ldap_back_map( &li->targets[ target]->oc_map, + bv, &mapped, 1 ); + if ( mapped.bv_val == NULL ) { + free( bv->bv_val ); + bv->bv_val = NULL; + if ( --last < 0 ) { + break; + } + *bv = attr->a_vals[ last ]; + attr->a_vals[ last ].bv_val = NULL; + i--; + } else if ( mapped.bv_val != bv->bv_val ) { + free( bv->bv_val ); + ber_dupbv( bv, &mapped ); + } + } + /* + * It is necessary to try to rewrite attributes with + * dn syntax because they might be used in ACLs as + * members of groups; since ACLs are applied to the + * rewritten stuff, no dn-based subecj clause could + * be used at the ldap backend side (see + * http://www.OpenLDAP.org/faq/data/cache/452.html) + * The problem can be overcome by moving the dn-based + * ACLs to the target directory server, and letting + * everything pass thru the ldap backend. + */ + } else if ( strcmp( attr->a_desc->ad_type->sat_syntax->ssyn_oid, + SLAPD_DN_SYNTAX ) == 0 ) { + int i; + for ( i = 0, bv = attr->a_vals; bv->bv_val; bv++,i++ ) { + char *newval; + rewriteSession(li->targets[ target ]->rwinfo, + "searchResult", bv->bv_val, + lc->conn, &newval, result); + if (result->type != SUCCESS) { + /* FIXME : Handle error */ + result->type = SUCCESS; + } else { + /* left as is */ + if ( newval == NULL ) { + break; + } +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, + "[rw] searchResult on " + "attr=%s: \"%s\" -> \"%s\"\n", + attr->a_desc->ad_type-> + sat_cname.bv_val, + bv->bv_val, newval ); +#else /* !NEW_LOGGING */ + Debug( LDAP_DEBUG_ARGS, + "rw> searchResult on attr=%s:" + " \"%s\" -> \"%s\"\n", + attr->a_desc->ad_type-> + sat_cname.bv_val, + bv->bv_val, newval ); +#endif /* !NEW_LOGGING */ + free( bv->bv_val ); + bv->bv_val = newval; + bv->bv_len = strlen( newval ); + } + } + } + *attrp = attr; + attrp = &attr->a_next; + } + return ent; +} + +static int +is_one_level_rdn( + const char *rdn, + int from +) +{ + for ( ; from--; ) { + if ( DN_SEPARATOR( rdn[ from ] ) ) { + return 0; + } + } + return 1; +} + +static struct metaconn* +metaConnect( + struct metainfo* li, + Connection* conn, + Operation* op, + int op_type, + struct berval* nbase, + struct exception* result) +{ + struct metaconn* lc; + result->type = SUCCESS; + lc = meta_back_getconn( li, conn, op, op_type, nbase, NULL ); + if (!lc) { + result->type = CONN_ERR; + return 0; + } + return lc; +} + +static void +add_filter_attrs( + AttributeName** new_attrs, + AttributeName* attrs, + AttributeName* filter_attrs ) +{ + struct berval all_user = { sizeof(LDAP_ALL_USER_ATTRIBUTES) -1, + LDAP_ALL_USER_ATTRIBUTES }; + + struct berval all_op = { sizeof(LDAP_ALL_OPERATIONAL_ATTRIBUTES) -1, + LDAP_ALL_OPERATIONAL_ATTRIBUTES}; + + int alluser = 0; + int allop = 0; + int i; + int count; + + /* duplicate attrs */ + for (count=0; attrs[count].an_name.bv_val; count++) + ; + *new_attrs = (AttributeName*)(malloc((count+1)*sizeof(AttributeName))); + for (i=0; iad_type) + && allop ) + continue; + else if (alluser) + continue; + *new_attrs = (AttributeName*)(realloc(*new_attrs, + (count+2)*sizeof(AttributeName))); + (*new_attrs)[count].an_name.bv_val = + filter_attrs[i].an_name.bv_val; + (*new_attrs)[count].an_name.bv_len = + filter_attrs[i].an_name.bv_len; + (*new_attrs)[count].an_desc = filter_attrs[i].an_desc; + (*new_attrs)[count+1].an_name.bv_val = NULL; + (*new_attrs)[count+1].an_name.bv_len = 0; + count++; + } +} + +static int +handleLdapResult( + struct metaconn* lc, + Operation* op, + int* msgid, Backend* be, + AttributeName* attrs, + int attrsonly, + int candidates, + int cacheable, + Entry*** entry_array, + int curr_limit, + int slimit, + struct exception* result) +{ + Entry *entry; + char *match = NULL, *err = NULL, *cache_ename = NULL; + int sres; + int mres = LDAP_SUCCESS; + int num_entries = 0, count, i, rc; + struct timeval tv = {0, 0}; + struct metasingleconn* lsc; + struct metainfo *li = ( struct metainfo * )be->be_private; + result->rc = 0; + result->type = SUCCESS; + + for ( count = 0, rc = 0; candidates > 0; ) { + int ab, gotit = 0; + + /* check for abandon */ + ab = op->o_abandon; + + for ( i = 0, lsc = lc->conns; !META_LAST(lsc); lsc++, i++ ) { + if ( lsc->candidate != META_CANDIDATE ) { + continue; + } + + if ( ab ) { + ldap_abandon( lsc->ld, msgid[ i ] ); + result->type = ABANDON_ERR; + break; + } + + if ( slimit > 0 && num_entries == slimit ) { + result->type = SLIMIT_ERR; + break; + } + + if ((entry = get_result_entry(be, lc, lsc, + msgid, i, &tv, result))) { + send_search_entry( be, lc->conn, op, entry, + attrs, attrsonly, NULL ); + if ((cacheable) && + (num_entries < curr_limit)) { + rewriteSession( li->rwinfo, + "cacheResult", + entry->e_name.bv_val, + lc->conn, + &cache_ename, result ); + free(entry->e_name.bv_val); + if (result->type != SUCCESS) { + return 0; + } + ber_str2bv(cache_ename, + strlen(cache_ename), + 0, &entry->e_name); + ber_dupbv(&entry->e_nname, + &entry->e_name); + *entry_array = (Entry**)realloc( + *entry_array, + (( num_entries+2 ) * + sizeof( Entry* ))); + (*entry_array)[num_entries] = entry; + (*entry_array)[num_entries+1] = NULL; + } + num_entries++; + gotit = 1; + } else if (result->type == REWRITING_ERR) { + return 0; + } else if (result->type == TIMEOUT_ERR) { + result->type = SUCCESS; + continue; + } else if (result->type == CREATE_ENTRY_ERR) { + break; + } else if (result->rc == -1) { + break; + } else { + sres = ldap_back_map_result(result->rc); + if (mres == LDAP_SUCCESS && + sres != LDAP_SUCCESS) { + mres = sres; + ldap_get_option(lsc->ld, + LDAP_OPT_ERROR_STRING, &err); + ldap_get_option(lsc->ld, + LDAP_OPT_MATCHED_DN, &match); + } + lsc->candidate = META_NOT_CANDIDATE; + candidates--; + result->type = SUCCESS; + } + } + switch (result->type) { + case RESULT_ERR: +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, + "ldap_result error, rc = -1\n", + 0, 0, 0 ); +#else /* !NEW_LOGGING */ + Debug( LDAP_DEBUG_ANY, "ldap_result error, rc = -1\n", + 0, 0, 0 ); +#endif /* !NEW_LOGGING */ + send_search_result( lc->conn, op, LDAP_OTHER, NULL, + NULL, NULL, NULL, num_entries ); + return 0; + case CREATE_ENTRY_ERR: +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, + "Error in parsing result \n", + 0, 0, 0 ); +#else /* !NEW_LOGGING */ + Debug( LDAP_DEBUG_ANY, "Error in parsing result \n", + 0, 0, 0 ); +#endif /* !NEW_LOGGING */ + send_search_result( lc->conn, op, LDAP_OTHER, NULL, + NULL, NULL, NULL, num_entries ); + result->type = RESULT_ERR; + return 0; + case SLIMIT_ERR: +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, "Size limit exceeded \n", + 0, 0, 0 ); +#else /* !NEW_LOGGING */ + Debug( LDAP_DEBUG_ANY, "Size limit exceeded \n", + 0, 0, 0 ); +#endif /* !NEW_LOGGING */ + send_search_result( lc->conn, op, + LDAP_SIZELIMIT_EXCEEDED, + NULL, NULL, NULL, NULL, num_entries ); + result->type = RESULT_ERR; + return 0; + case ABANDON_ERR: +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, + "search operation abandoned \n", + 0, 0, 0 ); +#else /* !NEW_LOGGING */ + Debug( LDAP_DEBUG_ANY, "search operation abandoned \n", + 0, 0, 0 ); +#endif /* !NEW_LOGGING */ + result->type = RESULT_ERR; + return 0; + default: + /* assert( 0 ); */ + } + if ( gotit == 0 ) { + tv.tv_sec = 0; + tv.tv_usec = 100000; + ldap_pvt_thread_yield(); + } else { + tv.tv_sec = 0; + tv.tv_usec = 0; + } + } + + send_search_result( lc->conn, op, mres, match, err, + NULL, NULL, num_entries ); + + if (err) + free(err); + + if (match) + free(match); + + result->type = (mres == LDAP_SUCCESS) ? SUCCESS : RESULT_ERR; + return num_entries; +} + +static Entry* +get_result_entry( + Backend* be, + struct metaconn* lc, + struct metasingleconn* lsc, + int* msgid, + int i, + struct timeval* tv, + struct exception* result) +{ + Entry* entry; + LDAPMessage *res, *e; + int rc; + int sres = LDAP_SUCCESS; + + rc = ldap_result( lsc->ld, msgid[ i ], + 0, tv, &res ); + + if ( rc == 0 ) { + result->type = TIMEOUT_ERR; + return NULL; + } else if ( rc == -1 ) { + result->rc = -1; + result->type = RESULT_ERR; + return NULL; + } else if ( rc == LDAP_RES_SEARCH_ENTRY ) { + e = ldap_first_entry( lsc->ld, res ); + entry = meta_create_entry(be, lc, i, e, result); + if (!entry) { + return NULL; + } + ldap_msgfree( res ); + result->type = SUCCESS; + return entry; + } else { + sres = ldap_result2error( lsc->ld, + res, 1 ); + result->rc = sres; + result->type = RESULT_ERR; + return NULL; + } +} + +static void +rewriteSession( + struct rewrite_info* info, + const char* rewriteContext, + const char* string, + const void* cookie, + char** base, + struct exception* result) +{ + int rc = rewrite_session(info, rewriteContext, string, cookie, base); + if (rc != REWRITE_REGEXEC_OK) { + result->rc = rc; + result->type = REWRITING_ERR; + + if (strcmp(rewriteContext, "searchBase") == 0) +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, + "Problem in rewriting search base\n", 0, 0, 0 ); +#else /* !NEW_LOGGING */ + Debug( LDAP_DEBUG_ANY, + "Problem in rewriting search base\n", 0, 0, 0 ); +#endif /* !NEW_LOGGING */ + if (strcmp(rewriteContext, "searchFilter") == 0) +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, + "Problem in rewriting search filter\n", + 0, 0, 0 ); +#else /* !NEW_LOGGING */ + Debug( LDAP_DEBUG_ANY, + "Problem in rewriting search filter\n", + 0, 0, 0 ); +#endif /* !NEW_LOGGING */ + if (strcmp(rewriteContext, "searchResult") == 0) +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, + "Problem in rewriting DN, or DN syntax " + "attributes of search result\n", 0, 0, 0 ); +#else /* !NEW_LOGGING */ + Debug( LDAP_DEBUG_ANY, + "Problem in rewriting DN, or DN syntax " + "attributes of search result\n", 0, 0, 0 ); +#endif /* !NEW_LOGGING */ + if (strcmp(rewriteContext, "cacheBase") == 0) +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, + "Problem in rewriting search base with " + "cache base\n", 0, 0, 0 ); +#else /* !NEW_LOGGING */ + Debug( LDAP_DEBUG_ANY, + "Problem in rewriting search base with " + "cache base\n", 0, 0, 0 ); +#endif /* !NEW_LOGGING */ + if (strcmp(rewriteContext, "cacheResult") == 0) +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, + "Problem in rewriting DN for cached entries\n", + 0, 0, 0 ); +#else /* !NEW_LOGGING */ + Debug( LDAP_DEBUG_ANY, + "Problem in rewriting DN for cached entries\n", + 0, 0, 0 ); +#endif /* !NEW_LOGGING */ + if (strcmp(rewriteContext, "cacheReturn") == 0) +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, + "Problem in rewriting DN for answerable " + "entries\n", 0, 0, 0 ); +#else /* !NEW_LOGGING */ + Debug( LDAP_DEBUG_ANY, + "Problem in rewriting DN for answerable " + "entries\n", 0, 0, 0 ); +#endif /* !NEW_LOGGING */ + } else { + result->type = SUCCESS; + } +} + +static int +get_attr_set( + AttributeName* attrs, + query_manager* qm, + int num ) +{ + int i; + for (i=0; iattr_sets[i].attrs)) + return i; + } + return -1; +} + +static int +attrscmp( + AttributeName* attrs_in, + AttributeName* attrs) +{ + int i, count1, count2; + if ((attrs_in==NULL) || (attrs==NULL)) + return 1; + for ( count1=0; + attrs_in && attrs_in[count1].an_name.bv_val != NULL; + count1++ ) + ; + for ( count2=0; + attrs && attrs[count2].an_name.bv_val != NULL; + count2++) + ; + if ( count1 != count2 ) + return 0; + + for ( i=0; iqm; + + result->type = SUCCESS; + query_uuid.bv_len = lutil_uuidstr(uuidbuf, sizeof(uuidbuf)); + query_uuid.bv_val = ch_strdup(uuidbuf); + +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, "UUID for query being added = %s\n", + uuidbuf, 0, 0 ); +#else /* !NEW_LOGGING */ + Debug( LDAP_DEBUG_ANY, "UUID for query being added = %s\n", + uuidbuf, 0, 0 ); +#endif /* !NEW_LOGGING */ + + for ( i=0; ( entry_array && (e=entry_array[i]) ); i++ ) { +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL2, "LOCKING REMOVE MUTEX\n", + 0, 0, 0 ); +#else /* !NEW_LOGGING */ + Debug( LDAP_DEBUG_NONE, "LOCKING REMOVE MUTEX\n", 0, 0, 0 ); +#endif /* !NEW_LOGGING */ + ldap_pvt_thread_mutex_lock(&cm->remove_mutex); +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL2, "LOCKED REMOVE MUTEX\n", 0, 0, 0); +#else /* !NEW_LOGGING */ + Debug( LDAP_DEBUG_NONE, "LOCKED REMOVE MUTEX\n", 0, 0, 0); +#endif /* !NEW_LOGGING */ + if ( cm->cache_size > (cm->thresh_hi) ) { + while(cm->cache_size > (cm->thresh_lo)) { + crpid = cache_replacement(qm); + if (crpid == NULL) { + result->type = REMOVE_ERR; + } else { + strcpy(crpuuid, crpid); + crp_uuid.bv_val = crpuuid; + crp_uuid.bv_len = strlen(crpuuid); +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, + "Removing query UUID %s\n", + crpuuid, 0, 0 ); +#else /* !NEW_LOGGING */ + Debug( LDAP_DEBUG_ANY, + "Removing query UUID %s\n", + crpuuid, 0, 0 ); +#endif /* !NEW_LOGGING */ + return_val = remove_query_data(be, conn, + &crp_uuid, result); +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, + "QUERY REMOVED, SIZE=%d\n", + return_val, 0, 0); +#else /* !NEW_LOGGING */ + Debug( LDAP_DEBUG_ANY, + "QUERY REMOVED, SIZE=%d\n", + return_val, 0, 0); +#endif /* !NEW_LOGGING */ + ldap_pvt_thread_mutex_lock( + &cm->cache_mutex ); + cm->total_entries -= result->rc; + cm->num_cached_queries--; +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, + "STORED QUERIES = %d\n", + cm->num_cached_queries, 0, 0 ); +#else /* !NEW_LOGGING */ + Debug( LDAP_DEBUG_ANY, + "STORED QUERIES = %d\n", + cm->num_cached_queries, 0, 0 ); +#endif /* !NEW_LOGGING */ + ldap_pvt_thread_mutex_unlock( + &cm->cache_mutex ); + cm->cache_size = (return_val > + cm->cache_size) ? + 0 : (cm->cache_size-return_val); +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, + "QUERY REMOVED, CACHE SIZE=" + "%d bytes %d entries\n", + cm->cache_size, + cm->total_entries, 0 ); +#else /* !NEW_LOGGING */ + Debug( LDAP_DEBUG_ANY, + "QUERY REMOVED, CACHE SIZE=" + "%d bytes %d entries\n", + cm->cache_size, + cm->total_entries, 0 ); +#endif /* !NEW_LOGGING */ + } + } + } + return_val = merge_entry(be, conn, e, &query_uuid, result); + cm->cache_size += return_val; +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, + "ENTRY ADDED/MERGED, CACHE SIZE=%d bytes\n", + cm->cache_size, 0, 0 ); +#else /* !NEW_LOGGING */ + Debug( LDAP_DEBUG_ANY, + "ENTRY ADDED/MERGED, CACHE SIZE=%d bytes\n", + cm->cache_size, 0, 0 ); +#endif /* !NEW_LOGGING */ +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL2, "UNLOCKING REMOVE MUTEX\n", + 0, 0, 0 ); +#else /* !NEW_LOGGING */ + Debug( LDAP_DEBUG_NONE, "UNLOCKING REMOVE MUTEX\n", 0, 0, 0 ); +#endif /* !NEW_LOGGING */ + ldap_pvt_thread_mutex_unlock(&cm->remove_mutex); +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL2, "UNLOCKED REMOVE MUTEX\n", + 0, 0, 0 ); +#else /* !NEW_LOGGING */ + Debug( LDAP_DEBUG_NONE, "UNLOCKED REMOVE MUTEX\n", 0, 0, 0 ); +#endif /* !NEW_LOGGING */ + if (result->type != SUCCESS) + return 0; + ldap_pvt_thread_mutex_lock(&cm->cache_mutex); + cm->total_entries += result->rc; +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, + "ENTRY ADDED/MERGED, SIZE=%d, CACHED ENTRIES=%d\n", + return_val, cm->total_entries, 0 ); +#else /* !NEW_LOGGING */ + Debug( LDAP_DEBUG_ANY, + "ENTRY ADDED/MERGED, SIZE=%d, CACHED ENTRIES=%d\n", + return_val, cm->total_entries, 0 ); +#endif /* !NEW_LOGGING */ + ldap_pvt_thread_mutex_unlock(&cm->cache_mutex); + } + ldap_pvt_thread_mutex_lock(&cm->cache_mutex); + cm->num_cached_queries++; +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, "STORED QUERIES = %d\n", + cm->num_cached_queries, 0, 0 ); +#else /* !NEW_LOGGING */ + Debug( LDAP_DEBUG_ANY, "STORED QUERIES = %d\n", + cm->num_cached_queries, 0, 0 ); +#endif /* !NEW_LOGGING */ + ldap_pvt_thread_mutex_unlock(&cm->cache_mutex); + + return query_uuid.bv_val; +} + +static int +is_temp_answerable( + int attr_set, + struct berval* tempstr, + query_manager* qm, + int template_id ) +{ + int i; + int* id_array; + char* str; + int result = 0; + i = qm->templates[template_id].attr_set_index; + str = qm->templates[template_id].querystr; + + if (attr_set == i) { + result = 1; + } else { + id_array = qm->attr_sets[attr_set].ID_array; + + while (*id_array != -1) { + if (*id_array == i) + result = 1; + id_array++; + } + } + if (!result) + return 0; + if (strcasecmp(str, tempstr->bv_val) == 0) + return 1; + return 0; +} + +static void +consistency_check(Backend* be, Backend* glue_be, Connection* conn) +{ + struct metainfo *li = ( struct metainfo * )be->be_private; + cache_manager* cm = li->cm; + query_manager* qm = cm->qm; + CachedQuery* query, *query_prev; + time_t curr_time; + struct berval uuid; + struct exception result; + int i, return_val; + QueryTemplate* templ; + + + for (i=0; qm->templates[i].querystr; i++) { + templ = qm->templates + i; + query = templ->query_last; + curr_time = slap_get_time(); + ldap_pvt_thread_mutex_lock(&cm->remove_mutex); + while (query && (query->expiry_time < curr_time)) { + ldap_pvt_thread_mutex_lock(&qm->lru_mutex); + remove_query(qm, query); + ldap_pvt_thread_mutex_unlock(&qm->lru_mutex); +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, "Lock CR index = %d\n", + i, 0, 0 ); +#else /* !NEW_LOGGING */ + Debug( LDAP_DEBUG_ANY, "Lock CR index = %d\n", + i, 0, 0 ); +#endif /* !NEW_LOGGING */ + ldap_pvt_thread_rdwr_wlock(&templ->t_rwlock); + remove_from_template(query, templ); +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, + "TEMPLATE %d QUERIES-- %d\n", + i, templ->no_of_queries, 0 ); +#else /* !NEW_LOGGING */ + Debug( LDAP_DEBUG_ANY, "TEMPLATE %d QUERIES-- %d\n", + i, templ->no_of_queries, 0 ); +#endif /* !NEW_LOGGING */ +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, "Unlock CR index = %d\n", + i, 0, 0 ); +#else /* !NEW_LOGGING */ + Debug( LDAP_DEBUG_ANY, "Unlock CR index = %d\n", + i, 0, 0 ); +#endif /* !NEW_LOGGING */ + ldap_pvt_thread_rdwr_wunlock(&templ->t_rwlock); + uuid.bv_val = query->q_uuid; + uuid.bv_len = strlen(query->q_uuid); + return_val = remove_query_data(glue_be, conn, + &uuid, &result); +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, + "STALE QUERY REMOVED, SIZE=%d\n", + return_val, 0, 0 ); +#else /* !NEW_LOGGING */ + Debug( LDAP_DEBUG_ANY, "STALE QUERY REMOVED, SIZE=%d\n", + return_val, 0, 0 ); +#endif /* !NEW_LOGGING */ + ldap_pvt_thread_mutex_lock(&cm->cache_mutex); + cm->total_entries -= result.rc; + cm->num_cached_queries--; +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, "STORED QUERIES = %d\n", + cm->num_cached_queries, 0, 0 ); +#else /* !NEW_LOGGING */ + Debug( LDAP_DEBUG_ANY, "STORED QUERIES = %d\n", + cm->num_cached_queries, 0, 0 ); +#endif /* !NEW_LOGGING */ + ldap_pvt_thread_mutex_unlock(&cm->cache_mutex); + cm->cache_size = (return_val > cm->cache_size) ? + 0: (cm->cache_size-return_val); +#ifdef NEW_LOGGING + LDAP_LOG( BACK_META, DETAIL1, + "STALE QUERY REMOVED, CACHE SIZE=%d bytes %d " + "entries\n", cm->cache_size, + cm->total_entries, 0 ); +#else /* !NEW_LOGGING */ + Debug( LDAP_DEBUG_ANY, + "STALE QUERY REMOVED, CACHE SIZE=%d bytes %d " + "entries\n", cm->cache_size, + cm->total_entries, 0 ); +#endif /* !NEW_LOGGING */ + query_prev = query; + query = query->prev; + free_query(query_prev); + } + ldap_pvt_thread_mutex_unlock(&cm->remove_mutex); + } +} + +static int +cache_back_sentry( + Backend* glue_be, + Connection* conn, + Operation* op, + Entry* e, + AttributeName* attrs, + int attrsonly, + LDAPControl** ctrls ) +{ + slap_callback* cb = op->o_callback; + Backend* be = (Backend*)(cb->sc_private); + struct metainfo *li = ( struct metainfo * )be->be_private; + + char* ename = NULL; + struct exception result; + struct berval dn; + struct berval ndn; + + dn = e->e_name; + ndn = e->e_nname; + + rewriteSession( li->rwinfo, + "cacheReturn", e->e_name.bv_val, conn, &ename, &result ); + ber_str2bv(ename, strlen(ename), 0, &e->e_name); + ber_dupbv(&e->e_nname, &e->e_name); + + op->o_callback = NULL; + send_search_entry(be, conn, op, e, attrs, attrsonly, ctrls); + + e->e_name = dn; + e->e_nname = ndn; + + op->o_callback = cb; + return 0; +} +#endif diff --git a/servers/slapd/back-meta/cache-substring.c b/servers/slapd/back-meta/cache-substring.c new file mode 100644 index 0000000000..8871093620 --- /dev/null +++ b/servers/slapd/back-meta/cache-substring.c @@ -0,0 +1,290 @@ +/* Copyright (c) 2003 by International Business Machines, Inc. + * + * International Business Machines, Inc. (hereinafter called IBM) grants + * permission under its copyrights to use, copy, modify, and distribute this + * Software with or without fee, provided that the above copyright notice and + * all paragraphs of this notice appear in all copies, and that the name of IBM + * not be used in connection with the marketing of any product incorporating + * the Software or modifications thereof, without specific, written prior + * permission. + * + * THE SOFTWARE IS PROVIDED "AS IS", AND IBM DISCLAIMS ALL WARRANTIES, + * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A + * PARTICULAR PURPOSE. IN NO EVENT SHALL IBM BE LIABLE FOR ANY SPECIAL, + * DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER ARISING + * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE, EVEN + * IF IBM IS APPRISED OF THE POSSIBILITY OF SUCH DAMAGES. + */ + +#include "portable.h" + +#include + +#include +#include +#include + +#include "slap.h" +#include "../back-ldap/back-ldap.h" +#include "back-meta.h" +#include "ldap_pvt.h" +#undef ldap_debug /* silence a warning in ldap-int.h */ +#include "ldap_log.h" +#include "../../../libraries/libldap/ldap-int.h" + +#include "slap.h" +#include "back-meta.h" + +#ifdef LDAP_CACHING +static char* invert_string(char* string); +static struct berval* merge_init_final(struct berval*, struct berval*, struct berval*); +static int strings_containment(struct berval* stored, struct berval* incoming); + +/* find and remove string2 from string1 + * from start if position = 1, + * from end if position = 3, + * from anywhere if position = 2 + */ + +int +find_and_remove(struct berval* ber1, struct berval* ber2, int position) +{ + char* temp; + int len; + int ret=0; + + char* arg1, *arg2; + char* string1=ber1->bv_val; + char* string2=ber2->bv_val; + + if (string2 == NULL) + return 1; + if (string1 == NULL) + return 0; + + if (position == 3) { + arg1 = invert_string(string1); + arg2 = invert_string(string2); + } else { + arg1 = string1; + arg2 = string2; + } + + temp = strstr(arg1, arg2); + len = strlen(arg2); + + if (!temp) + return 0; + + switch (position) { + case 1: + if (temp == arg1) { + string1 += len; + ret = 1; + } else { + ret = 0; + } + break; + case 2: + string1 = temp+len; + ret = 1; + break; + case 3: + temp = strstr(arg1, arg2); + len = strlen(arg2); + if (temp == arg1) { + /*arg1 += len;*/ + string1 = invert_string(arg1+len); + free(arg1); + free(arg2); + ret = 1; + } else { + free(arg1); + free(arg2); + ret = 0; + } + break; + } + temp = (char*) malloc( strlen( string1 ) + 1 ); + strcpy( temp, string1 ); + free( ber1->bv_val ); + ber1->bv_val = temp; + return ret; +} + +char* +invert_string( char* string ) +{ + int len = strlen(string); + int i; + + char* inverted = (char*)(malloc(len+1)); + + for (i=0; ibv_val) + count++; + if (final->bv_val) + count++; + + merged = (struct berval*)(malloc((count+1)*sizeof(struct berval))); + temp = merged; + + if (init->bv_val) { + ber_dupbv(temp, init); + temp++; + } + + for (i=0; ibv_val){ + ber_dupbv(temp, final); + temp++; + } + temp->bv_val = NULL; + temp->bv_len = 0; + return merged; +} + +int +strings_containment(struct berval* stored, struct berval* incoming) +{ + struct berval* element; + int k=0; + int j, rc = 0; + + for ( element=stored; element->bv_val != NULL; element++ ) { + for (j = k; incoming[j].bv_val != NULL; j++) { + if (find_and_remove(&(incoming[j]), element, 2)) { + k = j; + rc = 1; + break; + } + rc = 0; + } + if ( rc ) { + continue; + } else { + return 0; + } + } + return 1; +} + +int +substr_containment_substr(Filter* stored, Filter* incoming) +{ + int i; + int k=0; + int rc; + int any_count = 0; + + struct berval* init_incoming = (struct berval*)(malloc(sizeof(struct berval))); + struct berval* final_incoming = (struct berval*)(malloc(sizeof(struct berval))); + struct berval* any_incoming = NULL; + struct berval* remaining_incoming; + struct berval* any_element; + + if ((!(incoming->f_sub_initial.bv_val) && (stored->f_sub_initial.bv_val)) + || (!(incoming->f_sub_final.bv_val) && (stored->f_sub_final.bv_val))) + return 0; + + + ber_dupbv(init_incoming, &(incoming->f_sub_initial)); + ber_dupbv(final_incoming, &(incoming->f_sub_final)); + + if (incoming->f_sub_any) { + for ( any_count=0; incoming->f_sub_any[any_count].bv_val != NULL; + any_count++ ) + ; + + any_incoming = (struct berval*)malloc((any_count+1) * + sizeof(struct berval)); + + for (i=0; if_sub_any[i])); + } + any_incoming[any_count].bv_val = NULL; + any_incoming[any_count].bv_len = 0; + } + + if (find_and_remove(init_incoming, + &(stored->f_sub_initial), 1) && find_and_remove(final_incoming, + &(stored->f_sub_final), 3)) + { + if (stored->f_sub_any == NULL) { + rc = 1; + goto final; + } + remaining_incoming = merge_init_final(init_incoming, + any_incoming, final_incoming); + rc = strings_containment(stored->f_sub_any, remaining_incoming); + goto final; + } + rc = 0; +final: + /* + ber_bvfree(init_incoming); + ber_bvfree(final_incoming); + if (any_incoming) { + for (i=0; i < any_count; i++) + free(any_incoming[i].bv_val); + free(any_incoming); + } + */ + + return rc; +} + +int +substr_containment_equality(Filter* stored, Filter* incoming) +{ + + struct berval* incoming_val = (struct berval*)(malloc(2*sizeof(struct berval))); + int rc; + + ber_dupbv(incoming_val, &(incoming->f_av_value)); + incoming_val[1].bv_val = NULL; + incoming_val[1].bv_len = 0; + + if (find_and_remove(incoming_val, + &(stored->f_sub_initial), 1) && find_and_remove(incoming_val, + &(stored->f_sub_final), 3)) { + if (stored->f_sub_any == NULL){ + rc = 1; + goto final; + } + rc = strings_containment(stored->f_sub_any, incoming_val); + goto final; + } + rc=0; +final: + /* + if(incoming_val[0].bv_val) + free(incoming_val[0].bv_val); + free(incoming_val); + */ + return rc; +} +#endif diff --git a/servers/slapd/back-meta/cache-template.c b/servers/slapd/back-meta/cache-template.c new file mode 100644 index 0000000000..bad7b5b8d3 --- /dev/null +++ b/servers/slapd/back-meta/cache-template.c @@ -0,0 +1,247 @@ +/* Copyright (c) 2003 by International Business Machines, Inc. + * + * International Business Machines, Inc. (hereinafter called IBM) grants + * permission under its copyrights to use, copy, modify, and distribute this + * Software with or without fee, provided that the above copyright notice and + * all paragraphs of this notice appear in all copies, and that the name of IBM + * not be used in connection with the marketing of any product incorporating + * the Software or modifications thereof, without specific, written prior + * permission. + * + * THE SOFTWARE IS PROVIDED "AS IS", AND IBM DISCLAIMS ALL WARRANTIES, + * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A + * PARTICULAR PURPOSE. IN NO EVENT SHALL IBM BE LIABLE FOR ANY SPECIAL, + * DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER ARISING + * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE, EVEN + * IF IBM IS APPRISED OF THE POSSIBILITY OF SUCH DAMAGES. + */ + +#include "portable.h" + +#include + +#include +#include +#include + +#include "slap.h" +#include "../back-ldap/back-ldap.h" +#include "back-meta.h" +#undef ldap_debug /* silence a warning in ldap-int.h */ +#include "../../../libraries/libldap/ldap-int.h" + +#ifdef LDAP_CACHING +void +filter2template( + Filter *f, + struct berval *fstr, + AttributeName** filter_attrs, + int* filter_cnt, + struct exception* result ) +{ + int i; + Filter *p; + struct berval tmp; + ber_len_t len; + const char* text; + + /* + * FIXME: should we use an assert here? + */ + if ( f == NULL ) { + ber_str2bv( "No filter!", sizeof("No filter!")-1, 1, fstr ); + result->type = FILTER_ERR; + return; + } + + switch ( f->f_choice ) { + case LDAP_FILTER_EQUALITY: + fstr->bv_len = f->f_av_desc->ad_cname.bv_len + + ( sizeof("(=)") - 1 ); + fstr->bv_val = ch_malloc( fstr->bv_len + 1 ); + + snprintf( fstr->bv_val, fstr->bv_len + 1, "(%s=)", + f->f_av_desc->ad_cname.bv_val ); + + *filter_attrs = (AttributeName *)ch_realloc(*filter_attrs, + (*filter_cnt + 2)*sizeof(AttributeName)); + +#if 0 /* ? */ + ber_dupbv(&(*filter_attrs)[*filter_cnt].an_name, + &f->f_av_desc->ad_cname); +#endif + + (*filter_attrs)[*filter_cnt].an_name = f->f_av_desc->ad_cname; + (*filter_attrs)[*filter_cnt].an_desc = NULL; + slap_bv2ad(&f->f_av_desc->ad_cname, + &(*filter_attrs)[*filter_cnt].an_desc, &text); + (*filter_attrs)[*filter_cnt+1].an_name.bv_val = NULL; + (*filter_attrs)[*filter_cnt+1].an_name.bv_len = 0; + (*filter_cnt)++; + break; + case LDAP_FILTER_GE: + fstr->bv_len = f->f_av_desc->ad_cname.bv_len + + + ( sizeof("(>=)") - 1 ); + fstr->bv_val = ch_malloc( fstr->bv_len + 1 ); + + snprintf( fstr->bv_val, fstr->bv_len + 1, "(%s>=)", + f->f_av_desc->ad_cname.bv_val); + + *filter_attrs = (AttributeName *)ch_realloc(*filter_attrs, + (*filter_cnt + 2)*sizeof(AttributeName)); +#if 0 /* ? */ + ber_dupbv(&(*filter_attrs)[filter_cnt].an_name, + &f->f_av_desc->ad_cname); +#endif + + (*filter_attrs)[*filter_cnt].an_name = f->f_av_desc->ad_cname; + (*filter_attrs)[*filter_cnt].an_desc = NULL; + slap_bv2ad(&f->f_av_desc->ad_cname, + &(*filter_attrs)[*filter_cnt].an_desc, &text); + (*filter_attrs)[*filter_cnt+1].an_name.bv_val = NULL; + (*filter_attrs)[*filter_cnt+1].an_name.bv_len = 0; + (*filter_cnt)++; + + break; + + case LDAP_FILTER_LE: + fstr->bv_len = f->f_av_desc->ad_cname.bv_len + + + ( sizeof("(<=)") - 1 ); + fstr->bv_val = ch_malloc( fstr->bv_len + 1 ); + + snprintf( fstr->bv_val, fstr->bv_len + 1, "(%s<=)", + f->f_av_desc->ad_cname.bv_val); + + *filter_attrs = (AttributeName *)ch_realloc(*filter_attrs, + (*filter_cnt + 2)*sizeof(AttributeName)); +#if 0 /* ? */ + ber_dupbv(&(*filter_attrs)[filter_cnt].an_name, + &f->f_av_desc->ad_cname); +#endif + + (*filter_attrs)[*filter_cnt].an_name = f->f_av_desc->ad_cname; + (*filter_attrs)[*filter_cnt].an_desc = NULL; + slap_bv2ad(&f->f_av_desc->ad_cname, + &(*filter_attrs)[*filter_cnt].an_desc, &text); + (*filter_attrs)[*filter_cnt+1].an_name.bv_val = NULL; + (*filter_attrs)[*filter_cnt+1].an_name.bv_len = 0; + (*filter_cnt++); + + break; + + case LDAP_FILTER_APPROX: + fstr->bv_len = f->f_av_desc->ad_cname.bv_len + + + ( sizeof("(~=)") - 1 ); + fstr->bv_val = ch_malloc( fstr->bv_len + 1 ); + + snprintf( fstr->bv_val, fstr->bv_len + 1, "(%s~=)", + f->f_av_desc->ad_cname.bv_val); + + *filter_attrs = (AttributeName *)ch_realloc(*filter_attrs, + (*filter_cnt + 2)*sizeof(AttributeName)); + +#if 0 /* ? */ + ber_dupbv(&(*filter_attrs)[filter_cnt].an_name, + &f->f_av_desc->ad_cname); +#endif + + (*filter_attrs)[*filter_cnt].an_name = f->f_av_desc->ad_cname; + (*filter_attrs)[*filter_cnt].an_desc = NULL; + slap_bv2ad(&f->f_av_desc->ad_cname, + &(*filter_attrs)[*filter_cnt].an_desc, &text); + (*filter_attrs)[*filter_cnt+1].an_name.bv_val = NULL; + (*filter_attrs)[*filter_cnt+1].an_name.bv_len = 0; + (*filter_cnt)++; + + break; + + case LDAP_FILTER_SUBSTRINGS: + fstr->bv_len = f->f_sub_desc->ad_cname.bv_len + + ( sizeof("(=)") - 1 ); + fstr->bv_val = ch_malloc( fstr->bv_len + 1 ); + + snprintf( fstr->bv_val, fstr->bv_len + 1, "(%s=)", + f->f_sub_desc->ad_cname.bv_val ); + + *filter_attrs = (AttributeName *)ch_realloc(*filter_attrs, + (*filter_cnt + 2)*sizeof(AttributeName)); + +#if 0 /* ? */ + ber_dupbv(&(*filter_attrs)[filter_cnt].an_name, + &f->f_av_desc->ad_cname); +#endif + + (*filter_attrs)[*filter_cnt].an_name = f->f_av_desc->ad_cname; + (*filter_attrs)[*filter_cnt].an_desc = NULL; + slap_bv2ad(&f->f_av_desc->ad_cname, + &(*filter_attrs)[*filter_cnt].an_desc, &text); + (*filter_attrs)[*filter_cnt+1].an_name.bv_val = NULL; + (*filter_attrs)[*filter_cnt+1].an_name.bv_len = 0; + (*filter_cnt)++; + + break; + + case LDAP_FILTER_PRESENT: + fstr->bv_len = f->f_desc->ad_cname.bv_len + + ( sizeof("(=*)") - 1 ); + fstr->bv_val = ch_malloc( fstr->bv_len + 1 ); + + snprintf( fstr->bv_val, fstr->bv_len + 1, "(%s=*)", + f->f_desc->ad_cname.bv_val ); + + *filter_attrs = (AttributeName *)ch_realloc(*filter_attrs, + (*filter_cnt+2)*sizeof(AttributeName)); + +#if 0 /* ? */ + ber_dupbv(&(*filter_attrs)[filter_cnt].an_name, + &f->f_av_desc->ad_cname); +#endif + + (*filter_attrs)[*filter_cnt].an_name = f->f_desc->ad_cname; + (*filter_attrs)[*filter_cnt].an_desc = NULL; + slap_bv2ad(&f->f_desc->ad_cname, + &(*filter_attrs)[*filter_cnt].an_desc, &text); + (*filter_attrs)[*filter_cnt+1].an_name.bv_val = NULL; + (*filter_attrs)[*filter_cnt+1].an_name.bv_len = 0; + (*filter_cnt)++; + + break; + + case LDAP_FILTER_AND: + case LDAP_FILTER_OR: + case LDAP_FILTER_NOT: + fstr->bv_len = sizeof("(%)") - 1; + fstr->bv_val = ch_malloc( fstr->bv_len + 128 ); + + snprintf( fstr->bv_val, fstr->bv_len + 1, "(%c)", + f->f_choice == LDAP_FILTER_AND ? '&' : + f->f_choice == LDAP_FILTER_OR ? '|' : '!' ); + + for ( p = f->f_list; p != NULL; p = p->f_next ) { + len = fstr->bv_len; + + filter2template( p, &tmp, filter_attrs, filter_cnt, + result); + if (result->type != SUCCESS) { + return; + } + + fstr->bv_len += tmp.bv_len; + fstr->bv_val = ch_realloc( fstr->bv_val, + fstr->bv_len + 1 ); + + snprintf( &fstr->bv_val[len-1], tmp.bv_len + 2, + /*"("*/ "%s)", tmp.bv_val ); + + ch_free( tmp.bv_val ); + } + + break; + + default: + ber_str2bv( "(?=unknown)", sizeof("(?=unknown)")-1, 1, fstr ); + result->type = FILTER_ERR; + return; + } +} +#endif /* LDAP_CACHING */ diff --git a/servers/slapd/back-meta/cache.h b/servers/slapd/back-meta/cache.h new file mode 100644 index 0000000000..fc8621d655 --- /dev/null +++ b/servers/slapd/back-meta/cache.h @@ -0,0 +1,252 @@ +/* Copyright (c) 2003 by International Business Machines, Inc. + * + * International Business Machines, Inc. (hereinafter called IBM) grants + * permission under its copyrights to use, copy, modify, and distribute +this + * Software with or without fee, provided that the above copyright notice +and + * all paragraphs of this notice appear in all copies, and that the name +of IBM + * not be used in connection with the marketing of any product +incorporating + * the Software or modifications thereof, without specific, written prior + * permission. + * + * THE SOFTWARE IS PROVIDED "AS IS", AND IBM DISCLAIMS ALL WARRANTIES, + * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A + * PARTICULAR PURPOSE. IN NO EVENT SHALL IBM BE LIABLE FOR ANY SPECIAL, + * DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +ARISING + * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE, +EVEN + * IF IBM IS APPRISED OF THE POSSIBILITY OF SUCH DAMAGES. + */ + + +#ifndef META_CACHE_H +#define META_CACHE_H +#include "slap.h" + +#ifdef LDAP_CACHING +/* cache specific errors */ +enum type_of_result +{ + SUCCESS, + CONN_ERR, + RESULT_ERR, + FILTER_ERR, + REWRITING_ERR, + MERGE_ERR, + REMOVE_ERR, + SLIMIT_ERR, + ABANDON_ERR, + CREATE_ENTRY_ERR, + TIMEOUT_ERR, + SIZE_ERR, + GET_SIZE_ERR +}; + + +struct exception { + enum type_of_result type; + int rc; +}; + +/* query cache structs */ +/* query */ + +typedef struct Query_s { + Filter* filter; /* Search Filter */ + AttributeName* attrs; /* Projected attributes */ + struct berval base; /* Search Base */ + int scope; /* Search scope */ +} Query; + +/* struct representing a cached query */ +typedef struct cached_query_s { + Query query; /* LDAP query */ + char* q_uuid; /* query identifier */ + int template_id; /* template of the query */ + time_t expiry_time; /* time till the query is considered valid */ + struct cached_query_s *next; /* next query in the template */ + struct cached_query_s *prev; /* previous query in the template */ + struct cached_query_s *lru_up; /* previous query in the LRU list */ + struct cached_query_s *lru_down; /* next query in the LRU list */ +} CachedQuery; + +/* struct representing a query template + * e.g. template string = &(cn=)(mail=) + */ +typedef struct query_template_s { + char* querystr; /* Filter string corresponding to the QT */ + char* base; /* Search base */ + int attr_set_index; /* determines the projected attributes */ + + CachedQuery* query; /* most recent query cached for the template */ + CachedQuery* query_last; /* oldest query cached for the template */ + + int no_of_queries; /* Total number of queries in the template */ + long ttl; /* TTL for the queries of this template */ + ldap_pvt_thread_rdwr_t t_rwlock; /* Rd/wr lock for accessing queries in the template */ +} QueryTemplate; + +/* + * Represents a set of projected attributes and any + * supersets among all specified sets of attributes. + */ + +struct attr_set { + AttributeName* attrs; /* specifies the set */ + int count; /* number of attributes */ + int* ID_array; /* array of indices of supersets of 'attrs' */ +}; + +struct query_manager_s; + +/* prototypes for functions for 1) query containment + * 2) query addition, 3) cache replacement + */ +typedef int (*QCfunc)(struct query_manager_s*, Query*, int ); +typedef void (*AddQueryfunc)(struct query_manager_s*, Query*, int, char*, struct exception* ); +typedef char* (*CRfunc)(struct query_manager_s* ); + +/* LDAP query cache */ +typedef struct query_manager_s { + struct attr_set* attr_sets; /* possible sets of projected attributes */ + QueryTemplate* templates; /* cacheable templates */ + + CachedQuery* lru_top; /* top and bottom of LRU list */ + CachedQuery* lru_bottom; + + ldap_pvt_thread_mutex_t lru_mutex; /* mutex for accessing LRU list */ + + /* Query cache methods */ + QCfunc qcfunc; /* Query containment*/ + CRfunc crfunc; /* cache replacement */ + AddQueryfunc addfunc; /* add query */ +} query_manager; + +/* LDAP query cache manager */ +typedef struct cache_manager_s { + unsigned long cache_size; /* current cache size (bytes) */ + unsigned long thresh_hi; /* cache capacity (bytes) */ + unsigned long thresh_lo; /* lower threshold for cache replacement */ + unsigned long num_cached_queries; /* total number of cached queries */ + unsigned long max_queries; /* upper bound on # of cached queries */ + int caching; + + int numattrsets; /* number of attribute sets */ + int numtemplates; /* number of cacheable templates */ + int total_entries; /* total number of entries cached */ + int num_entries_limit; /* max # of entries in a cacheable query */ + int threads; /* number of threads currently in meta_back_search */ + + int consistency_cycle_time; /* interval between successive consistency checks (sec) */ + int consistency_time; /* time when consistency check was last performed (sec) */ + + ldap_pvt_thread_mutex_t cache_mutex; + ldap_pvt_thread_mutex_t remove_mutex; + ldap_pvt_thread_mutex_t consistency_mutex; + + query_manager* qm; /* query cache managed by the cache manager */ +} cache_manager; + +/* search-cache.c */ +int +meta_back_cache_search( + Backend *be, + Connection *conn, + Operation *op, + struct berval *base, + struct berval *nbase, + int scope, + int deref, + int slimit, + int tlimit, + Filter *filt, + struct berval *filterstr, + AttributeName *attributes, + int attrsonly +); + +/* config-cache.c */ +int +meta_back_cache_config( + BackendDB *be, + const char *fname, + int lineno, + int argc, + char **argv +); + +/* query-cache.c */ +int query_containment(query_manager*, Query*, int); +void add_query(query_manager*, Query*, int, char*, struct exception*); +char* cache_replacement(query_manager*); +void remove_from_template (CachedQuery*, QueryTemplate*); +void remove_query (query_manager*, CachedQuery*); +void free_query (CachedQuery*); + +/* substring.c */ +int substr_containment_substr(Filter*, Filter*); +int substr_containment_equality(Filter*, Filter*); + +/* template.c */ +void +filter2template( Filter *f, + struct berval *fstr, + AttributeName** filter_attrs, + int* filter_cnt, + struct exception* result +); + +/* merge.c */ + +int +merge_entry (Backend* be, + Connection* conn, + Entry* e, + struct berval* query_uuid, + struct exception* result +); + +int +get_entry_size(Entry* e, + int size_init, + struct exception* result +); + +void +callback_null_response( Connection *conn, + Operation *o, + ber_tag_t tag, + ber_int_t msgid, + ber_int_t err, + const char *matched, + const char *text, + BerVarray ref, + const char *resoid, + struct berval *resdata, + struct berval *sasldata, + LDAPControl **c +); + +void callback_null_sresult( Connection *conn, + Operation *o, + ber_int_t err, + const char *matched, + const char *text, + BerVarray refs, + LDAPControl **c, + int nentries +); + +/* remove.c */ +int +remove_query_data (Backend* be, + Connection* conn, + struct berval* query_uuid, + struct exception* result +); +#endif +#endif diff --git a/servers/slapd/back-meta/config.c b/servers/slapd/back-meta/config.c index 00e430566f..87ae4e76f4 100644 --- a/servers/slapd/back-meta/config.c +++ b/servers/slapd/back-meta/config.c @@ -73,7 +73,7 @@ #include "slap.h" #include "../back-ldap/back-ldap.h" -#undef ldap_debug /* silence a warning in ldap-int.h */ +#undef ldap_debug /* silence a warning in ldap-int.h */ #include "../../../libraries/libldap/ldap-int.h" #include "back-meta.h" @@ -503,9 +503,17 @@ meta_back_db_config( int i = li->ntargets-1; if ( i < 0 ) { +#ifndef LDAP_CACHING fprintf( stderr, "%s: line %d: need \"uri\" directive first\n", fname, lineno ); +#else /* LDAP_CACHING */ + if ( strcasecmp( argv[0], "rewriteEngine" ) == 0 ) { + li->rwinfo = rewrite_info_init( REWRITE_MODE_USE_DEFAULT ); + } + return rewrite_parse(li->rwinfo, fname, lineno, + argc, argv); +#endif /* LDAP_CACHING */ } return rewrite_parse( li->targets[ i ]->rwinfo, fname, lineno, @@ -520,6 +528,12 @@ meta_back_db_config( fname, lineno, argc, argv ); /* anything else */ } else { +#ifdef LDAP_CACHING + if ( meta_back_cache_config( be, fname, lineno, argc, argv ) == 0 ) { + return 0; + } +#endif /* LDAP_CACHING */ + fprintf( stderr, "%s: line %d: unknown directive \"%s\" in meta database definition" " (ignored)\n", diff --git a/servers/slapd/back-meta/init.c b/servers/slapd/back-meta/init.c index 5d3f2b8f20..1aa4054ed5 100644 --- a/servers/slapd/back-meta/init.c +++ b/servers/slapd/back-meta/init.c @@ -137,6 +137,59 @@ meta_back_db_init( { struct metainfo *li; +#ifdef LDAP_CACHING + struct rewrite_info *rwinfo; + cache_manager *cm; + query_manager *qm; + + rwinfo = rewrite_info_init( REWRITE_MODE_USE_DEFAULT ); + if ( rwinfo == NULL ) { + return -1; + } + + cm = (cache_manager *)ch_malloc(sizeof(cache_manager)); + if ( cm == NULL ) { + rewrite_info_delete( rwinfo ); + return -1; + } + + qm = (query_manager*)ch_malloc(sizeof(query_manager)); + if ( qm == NULL ) { + rewrite_info_delete( rwinfo ); + ch_free( cm ); + return -1; + } + + cm->caching = 0; + cm->qm = qm; + cm->numattrsets = 0; + cm->numtemplates = 0; + cm->num_entries_limit = 5; + cm->cache_size = 0; + cm->thresh_hi = 500000; + cm->thresh_lo = 700000; + cm->num_cached_queries = 0; + cm->total_entries = 0; + cm->max_queries = 10000; + cm->threads = 0; + cm->consistency_time = slap_get_time(); + cm->consistency_cycle_time = 1000; + + qm->attr_sets = NULL; + qm->templates = NULL; + qm->lru_top = NULL; + qm->lru_bottom = NULL; + + qm->qcfunc = query_containment; + qm->crfunc = cache_replacement; + qm->addfunc = add_query; + ldap_pvt_thread_mutex_init(&qm->lru_mutex); + + ldap_pvt_thread_mutex_init(&cm->cache_mutex); + ldap_pvt_thread_mutex_init(&cm->remove_mutex); + ldap_pvt_thread_mutex_init( &cm->consistency_mutex ); +#endif /* LDAP_CACHING */ + li = ch_calloc( 1, sizeof( struct metainfo ) ); if ( li == NULL ) { return -1; @@ -147,6 +200,11 @@ meta_back_db_init( * this may change */ li->defaulttarget = META_DEFAULT_TARGET_NONE; +#ifdef LDAP_CACHING + li->cm = cm; + li->rwinfo = rwinfo; + /* FIXME: what about qm ? */ +#endif /* LDAP_CACHING */ ldap_pvt_thread_mutex_init( &li->conn_mutex ); ldap_pvt_thread_mutex_init( &li->cache.mutex ); diff --git a/servers/slapd/back-meta/search.c b/servers/slapd/back-meta/search.c index 98ebbafbaa..15ebc56946 100644 --- a/servers/slapd/back-meta/search.c +++ b/servers/slapd/back-meta/search.c @@ -129,6 +129,16 @@ meta_back_search( struct slap_limits_set *limit = NULL; int isroot = 0; +#ifdef LDAP_CACHING + cache_manager* cm = li->cm; + + if (cm->caching) { + return meta_back_cache_search(be, conn, op, base, nbase, + scope, deref, slimit, tlimit, + filter, filterstr, attrs, attrsonly); + } +#endif /* LDAP_CACHING */ + /* * controls are set in ldap_back_dobind() * diff --git a/servers/slapd/schema_prep.c b/servers/slapd/schema_prep.c index cadc70d3e9..87aaf2437d 100644 --- a/servers/slapd/schema_prep.c +++ b/servers/slapd/schema_prep.c @@ -475,6 +475,18 @@ static struct slap_schema_ad_map { NULL, NULL, NULL, NULL, NULL, offsetof(struct slap_internal_schema, si_ad_superiorUUID) }, +#ifdef LDAP_CACHING + /* LDAP cache specific operation attribute */ + { "queryid", "( 2.5.18.16 NAME 'queryid' " + "DESC 'list of queries the entry belongs to' " + "EQUALITY octetStringMatch " + "SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64} " + "NO-USER-MODIFICATION USAGE directoryOperation )", + NULL, 0, /* SLAP_AT_HIDE or SLAP_AT_NONE ? */ + NULL, NULL, NULL, NULL, NULL, + offsetof(struct slap_internal_schema, si_ad_queryid) }, +#endif /* LDAP_CACHING */ + /* root DSE attributes */ { "altServer", "( 1.3.6.1.4.1.1466.101.120.6 NAME 'altServer' " "DESC 'RFC2252: alternative servers' " diff --git a/servers/slapd/slap.h b/servers/slapd/slap.h index 4f75559736..2c712bb977 100644 --- a/servers/slapd/slap.h +++ b/servers/slapd/slap.h @@ -730,6 +730,11 @@ struct slap_internal_schema { AttributeDescription *si_ad_entryCSN; AttributeDescription *si_ad_superiorUUID; +#ifdef LDAP_CACHING + /* LDAP cache specific operational attribute */ + AttributeDescription *si_ad_queryid; +#endif /* LDAP_CACHING */ + /* root DSE attribute descriptions */ AttributeDescription *si_ad_altServer; AttributeDescription *si_ad_namingContexts; @@ -1826,6 +1831,10 @@ typedef struct slap_op { LDAP_STAILQ_ENTRY(slap_op) o_next; /* next operation in list */ ValuesReturnFilter *vrFilter; /* Structure represents ValuesReturnFilter */ +#ifdef LDAP_CACHING + char o_caching_on; +#endif /*LDAP_CACHING */ + #ifdef LDAP_SLAPI void *o_pb; /* NS-SLAPI plugin */ #endif -- 2.39.5