From 795841b5a4e240040b1021fae89e90f68cbb02a6 Mon Sep 17 00:00:00 2001 From: Pierangelo Masarati Date: Sat, 27 May 2006 19:54:27 +0000 Subject: [PATCH] implement proxy quarantine (ITS#4569) --- doc/man/man5/slapd-ldap.5 | 38 +++-- doc/man/man5/slapd-meta.5 | 18 +++ servers/slapd/back-ldap/back-ldap.h | 26 +++- servers/slapd/back-ldap/bind.c | 119 +++++++++++++++- servers/slapd/back-ldap/config.c | 200 ++++++++++++++++++++++++++- servers/slapd/back-ldap/extended.c | 20 +++ servers/slapd/back-ldap/init.c | 3 + servers/slapd/back-ldap/proto-ldap.h | 12 ++ servers/slapd/back-ldap/search.c | 6 + servers/slapd/back-meta/add.c | 4 + servers/slapd/back-meta/back-meta.h | 16 ++- servers/slapd/back-meta/bind.c | 20 ++- servers/slapd/back-meta/config.c | 44 ++++++ servers/slapd/back-meta/conn.c | 131 ++++++++++++++++-- servers/slapd/back-meta/delete.c | 4 + servers/slapd/back-meta/init.c | 10 ++ servers/slapd/back-meta/modify.c | 4 + servers/slapd/back-meta/modrdn.c | 4 + servers/slapd/back-meta/search.c | 7 +- 19 files changed, 650 insertions(+), 36 deletions(-) diff --git a/doc/man/man5/slapd-ldap.5 b/doc/man/man5/slapd-ldap.5 index 2df474253c..79eb9349e3 100644 --- a/doc/man/man5/slapd-ldap.5 +++ b/doc/man/man5/slapd-ldap.5 @@ -343,10 +343,6 @@ used by the client, otherwise the requested protocol is used. The proxy returns \fIunwillingToPerform\fP if an operation that is incompatible with the requested protocol is attempted. -.TP -.B single\-conn {NO|yes} -Discards current cached connection when the client rebinds. - .TP .B proxy\-whoami {NO|yes} Turns on proxying of the WhoAmI extended operation. If this option is @@ -356,12 +352,34 @@ request will be forwarded to the remote LDAP server. Other sessions will be handled by the local slapd, as before. This option is mainly useful in conjunction with Proxy Authorization. +.TP +.B quarantine ,[;,[...]] +Turns on quarantine of URIs that returned +.IR LDAP_UNAVAILABLE , +so that an attempt to reconnect only occurs at given intervals instead +of any time a client requests an operation. +The pattern is: retry only after at least +.I interval +seconds elapsed since last attempt, for exactly +.I num +times; then use the next pattern. +If +.I num +for the last pattern is "\fB+\fP", it retries forever; otherwise, +no more retries occur. +The process can be restarted by resetting the \fIolcDbQuarantine\fP +attribute of the database entry in the configuration backend. + .TP .B rebind-as-user {NO|yes} If this option is given, the client's bind credentials are remembered for rebinds when chasing referrals. Useful when \fBchase-referrals\fP is set to \fByes\fP, useless otherwise. +.TP +.B single\-conn {NO|yes} +Discards current cached connection when the client rebinds. + .TP .B t-f-support {NO|yes|discover} enable if the remote server supports absolute filters @@ -400,12 +418,6 @@ as a side-effect, some of the traditional directives have been deprecated and should be no longer used, as they might disappear in future releases. -.TP -.B server -this directive is no longer supported. Use the -.B uri -directive as described above. - .TP .B acl-authcDN "" DN which is used to query the target server for acl checking; it @@ -476,6 +488,12 @@ arg of .BR idassert-bind , and will be dismissed in the future. +.TP +.B server +this directive is no longer supported. Use the +.B uri +directive as described above. + .TP .B suffixmassage, map, rewrite* These directives are no longer supported by back-ldap; their diff --git a/doc/man/man5/slapd-meta.5 b/doc/man/man5/slapd-meta.5 index 8beab5adff..58d6760439 100644 --- a/doc/man/man5/slapd-meta.5 +++ b/doc/man/man5/slapd-meta.5 @@ -143,6 +143,24 @@ This directive, when set to causes the authentication to the remote servers with the pseudo-root identity to be deferred until actually needed by subsequent operations. +.TP +.B quarantine ,[;,[...]] +Turns on quarantine of URIs that returned +.IR LDAP_UNAVAILABLE , +so that an attempt to reconnect only occurs at given intervals instead +of any time a client requests an operation. +The pattern is: retry only after at least +.I interval +seconds elapsed since last attempt, for exactly +.I num +times; then use the next pattern. +If +.I num +for the last pattern is "\fB+\fP", it retries forever; otherwise, +no more retries occur. +This directive must appear before any target specification; +it affects all targets with the same pattern. + .TP .B rebind-as-user {NO|yes} If this option is given, the client's bind credentials are remembered diff --git a/servers/slapd/back-ldap/back-ldap.h b/servers/slapd/back-ldap/back-ldap.h index 197b745576..f8e6bde9a0 100644 --- a/servers/slapd/back-ldap/back-ldap.h +++ b/servers/slapd/back-ldap/back-ldap.h @@ -122,13 +122,26 @@ typedef struct ldap_avl_info_t { Avlnode *lai_tree; } ldap_avl_info_t; +typedef struct slap_retry_info_t { + time_t *ri_interval; + int *ri_num; + int ri_idx; + int ri_count; + time_t ri_last; + +#define SLAP_RETRYNUM_FOREVER (-1) /* retry forever */ +#define SLAP_RETRYNUM_TAIL (-2) /* end of retrynum array */ +#define SLAP_RETRYNUM_VALID(n) ((n) >= SLAP_RETRYNUM_FOREVER) /* valid retrynum */ +#define SLAP_RETRYNUM_FINITE(n) ((n) > SLAP_RETRYNUM_FOREVER) /* not forever */ +} slap_retry_info_t; + typedef struct ldapinfo_t { /* li_uri: the string that goes into ldap_initialize() * TODO: use li_acl.sb_uri instead */ - char *li_uri; + char *li_uri; /* li_bvuri: an array of each single URI that is equivalent; * to be checked for the presence of a certain item */ - BerVarray li_bvuri; + BerVarray li_bvuri; ldap_pvt_thread_mutex_t li_uri_mutex; LDAP_REBIND_PROC *li_rebind_f; @@ -231,6 +244,15 @@ typedef struct ldapinfo_t { ldap_avl_info_t li_conninfo; + slap_retry_info_t li_quarantine; + /* NOTE: quarantine uses the connection mutex */ + sig_atomic_t li_isquarantined; +#define LDAP_BACK_FQ_NO (0) +#define LDAP_BACK_FQ_YES (1) +#define LDAP_BACK_FQ_RETRYING (2) + +#define LDAP_BACK_QUARANTINE(li) ( (li)->li_quarantine.ri_num != NULL ) + time_t li_network_timeout; time_t li_conn_ttl; time_t li_idle_timeout; diff --git a/servers/slapd/back-ldap/bind.c b/servers/slapd/back-ldap/bind.c index 74626b493c..0072aa17ee 100644 --- a/servers/slapd/back-ldap/bind.c +++ b/servers/slapd/back-ldap/bind.c @@ -600,6 +600,37 @@ ldap_back_getconn( Operation *op, SlapReply *rs, ldap_back_send_t sendok ) lc_curr = { 0 }; int refcnt = 1, binding = 1; + /* if the server is quarantined, and + * - the current interval did not expire yet, or + * - no more retries should occur, + * don't return the connection */ + if ( li->li_isquarantined ) { + slap_retry_info_t *ri = &li->li_quarantine; + int dont_retry = 1; + + ldap_pvt_thread_mutex_lock( &li->li_conninfo.lai_mutex ); + if ( li->li_isquarantined == LDAP_BACK_FQ_YES ) { + dont_retry = ( ri->ri_num[ ri->ri_idx ] == SLAP_RETRYNUM_TAIL + || slap_get_time() < ri->ri_last + ri->ri_interval[ ri->ri_idx ] ); + if ( !dont_retry ) { + Debug( LDAP_DEBUG_ANY, + "%s: ldap_back_getconn quarantine " + "retry block #%d try #%d.\n", + op->o_log_prefix, ri->ri_idx, ri->ri_count ); + li->li_isquarantined = LDAP_BACK_FQ_RETRYING; + } + } + ldap_pvt_thread_mutex_unlock( &li->li_conninfo.lai_mutex ); + + if ( dont_retry ) { + rs->sr_err = LDAP_UNAVAILABLE; + if ( op->o_conn && ( sendok & LDAP_BACK_SENDERR ) ) { + send_ldap_result( op, rs ); + } + return NULL; + } + } + /* Internal searches are privileged and shared. So is root. */ if ( op->o_do_not_cache || be_isroot( op ) ) { LDAP_BACK_CONN_ISPRIV_SET( &lc_curr ); @@ -632,6 +663,7 @@ retry_lock: ldap_pvt_thread_yield(); goto retry_lock; } + refcnt = ++lc->lc_refcnt; binding = ++lc->lc_binding; } @@ -742,7 +774,6 @@ retry_lock: } } else { - char buf[ SLAP_TEXT_BUFLEN ]; int expiring = 0; if ( ( li->li_idle_timeout != 0 && op->o_time > lc->lc_time + li->li_idle_timeout ) @@ -760,13 +791,14 @@ retry_lock: } if ( LogTest( LDAP_DEBUG_TRACE ) ) { + char buf[ SLAP_TEXT_BUFLEN ]; + snprintf( buf, sizeof( buf ), "conn %p fetched refcnt=%u binding=%u%s", (void *)lc, refcnt, binding, expiring ? " expiring" : "" ); Debug( LDAP_DEBUG_TRACE, "=>ldap_back_getconn: %s.\n", buf, 0, 0 ); } - } #ifdef HAVE_TLS @@ -802,6 +834,67 @@ ldap_back_release_conn_lock( } } +void +ldap_back_quarantine( + Operation *op, + SlapReply *rs, + int dolock ) +{ + ldapinfo_t *li = (ldapinfo_t *)op->o_bd->be_private; + + slap_retry_info_t *ri = &li->li_quarantine; + + if ( dolock ) { + ldap_pvt_thread_mutex_lock( &li->li_conninfo.lai_mutex ); + } + + if ( rs->sr_err == LDAP_UNAVAILABLE ) { + switch ( li->li_isquarantined ) { + case LDAP_BACK_FQ_NO: + Debug( LDAP_DEBUG_ANY, + "%s: ldap_back_quarantine enter.\n", + op->o_log_prefix, 0, 0 ); + + ri->ri_idx = 0; + ri->ri_count = 0; + break; + + case LDAP_BACK_FQ_RETRYING: + Debug( LDAP_DEBUG_ANY, + "%s: ldap_back_quarantine block #%d try #%d failed.\n", + op->o_log_prefix, ri->ri_idx, ri->ri_count ); + + ++ri->ri_count; + if ( ri->ri_num[ ri->ri_idx ] != SLAP_RETRYNUM_FOREVER + && ri->ri_count == ri->ri_num[ ri->ri_idx ] ) + { + ri->ri_count = 0; + ++ri->ri_idx; + } + break; + + default: + break; + } + + li->li_isquarantined = LDAP_BACK_FQ_YES; + ri->ri_last = slap_get_time(); + + } else if ( li->li_isquarantined != LDAP_BACK_FQ_NO ) { + Debug( LDAP_DEBUG_ANY, + "%s: ldap_back_quarantine exit.\n", + op->o_log_prefix, ri->ri_idx, ri->ri_count ); + + ri->ri_count = 0; + ri->ri_idx = 0; + li->li_isquarantined = LDAP_BACK_FQ_NO; + } + + if ( dolock ) { + ldap_pvt_thread_mutex_unlock( &li->li_conninfo.lai_mutex ); + } +} + /* * ldap_back_dobind * @@ -929,7 +1022,7 @@ retry_lock:; if ( li->li_acl_secprops != NULL ) { rc = ldap_set_option( lc->lc_ld, - LDAP_OPT_X_SASL_SECPROPS, li->li_acl_secprops); + LDAP_OPT_X_SASL_SECPROPS, li->li_acl_secprops ); if ( rc != LDAP_OPT_SUCCESS ) { Debug( LDAP_DEBUG_ANY, "Error: ldap_set_option " @@ -1012,6 +1105,11 @@ retry:; } } + if ( LDAP_BACK_QUARANTINE( li ) ) { + ldap_back_quarantine( op, rs, dolock ); + } + + /* FIXME: one binding-- too many? */ lc->lc_binding--; ldap_back_freeconn( op, lc, dolock ); rs->sr_err = slap_map_api2result( rs ); @@ -1025,6 +1123,10 @@ retry:; } done:; + if ( LDAP_BACK_QUARANTINE( li ) ) { + ldap_back_quarantine( op, rs, dolock ); + } + lc->lc_binding--; LDAP_BACK_CONN_BINDING_CLEAR( lc ); rc = LDAP_BACK_CONN_ISBOUND( lc ); @@ -1152,6 +1254,8 @@ ldap_back_op_result( time_t timeout, ldap_back_send_t sendok ) { + ldapinfo_t *li = (ldapinfo_t *)op->o_bd->be_private; + char *match = NULL; LDAPMessage *res = NULL; char *text = NULL; @@ -1247,6 +1351,9 @@ retry:; rs->sr_matched = match; } } + if ( LDAP_BACK_QUARANTINE( li ) ) { + ldap_back_quarantine( op, rs, 1 ); + } if ( op->o_conn && ( ( sendok & LDAP_BACK_SENDOK ) || ( ( sendok & LDAP_BACK_SENDERR ) && rs->sr_err != LDAP_SUCCESS ) ) ) @@ -1307,11 +1414,15 @@ ldap_back_retry( ldapconn_t **lcp, Operation *op, SlapReply *rs, ldap_back_send_ rc = ldap_back_prepare_conn( lcp, op, rs, sendok ); if ( rc != LDAP_SUCCESS ) { rc = 0; + /* freeit, because lc_refcnt == 1 */ + (void)ldap_back_conn_free( *lcp ); *lcp = NULL; } else { rc = ldap_back_dobind_int( *lcp, op, rs, sendok, 0, 0 ); - if ( rc == 0 ) { + if ( rc == 0 && *lcp != NULL ) { + /* freeit, because lc_refcnt == 1 */ + (void)ldap_back_conn_free( *lcp ); *lcp = NULL; } } diff --git a/servers/slapd/back-ldap/config.c b/servers/slapd/back-ldap/config.c index 02b475ca76..d0ebdaa530 100644 --- a/servers/slapd/back-ldap/config.c +++ b/servers/slapd/back-ldap/config.c @@ -66,6 +66,7 @@ enum { LDAP_BACK_CFG_VERSION, LDAP_BACK_CFG_SINGLECONN, LDAP_BACK_CFG_CANCEL, + LDAP_BACK_CFG_QUARANTINE, LDAP_BACK_CFG_REWRITE, LDAP_BACK_CFG_LAST @@ -268,6 +269,14 @@ static ConfigTable ldapcfg[] = { "SYNTAX OMsDirectoryString " "SINGLE-VALUE )", NULL, NULL }, + { "quarantine", "retrylist", 2, 0, 0, + ARG_MAGIC|LDAP_BACK_CFG_QUARANTINE, + ldap_back_cf_gen, "( OLcfgDbAt:3.21 " + "NAME 'olcDbQuarantine' " + "DESC 'Quarantine database if connection fails and retry according to rule' " + "SYNTAX OMsDirectoryString " + "SINGLE-VALUE )", + NULL, NULL }, { "suffixmassage", "[virtual]> ri_interval = ch_calloc( sizeof( time_t ), i + 1 ); + ri->ri_num = ch_calloc( sizeof( int ), i + 1 ); + + for ( i = 0; retrylist[ i ] != NULL; i++ ) { + char *sep = strchr( retrylist[ i ], ',' ); + + if ( sep == NULL ) { + snprintf( buf, buflen, + "missing comma in retry pattern #%d \"%s\"", + i, retrylist[ i ] ); + rc = 1; + goto done; + } + + *sep++ = '\0'; + + if ( lutil_atol( &ri->ri_interval[ i ], retrylist[ i ] ) ) { + snprintf( buf, buflen, + "unable to parse interval #%d \"%s\"", + i, retrylist[ i ] ); + rc = 1; + goto done; + } + + if ( strcmp( sep, "+" ) == 0 ) { + if ( retrylist[ i + 1 ] != NULL ) { + snprintf( buf, buflen, + "extra cruft after retry pattern " + "#%d \"%s,+\" with \"forever\" mark", + i, retrylist[ i ] ); + rc = 1; + goto done; + } + ri->ri_num[ i ] = SLAP_RETRYNUM_FOREVER; + + } else if ( lutil_atoi( &ri->ri_num[ i ], sep ) ) { + snprintf( buf, buflen, + "unable to parse retry num #%d \"%s\"", + i, sep ); + rc = 1; + goto done; + } + } + + ri->ri_num[ i ] = SLAP_RETRYNUM_TAIL; + + ri->ri_idx = 0; + ri->ri_count = 0; + ri->ri_last = (time_t)(-1); + +done:; + ldap_charray_free( retrylist ); + + if ( rc ) { + slap_retry_info_destroy( ri ); + } + + return rc; +} + +int +slap_retry_info_unparse( + slap_retry_info_t *ri, + struct berval *bvout ) +{ + int i; + char buf[ BUFSIZ * 2 ], + *ptr = buf; + struct berval bv = BER_BVNULL; + + assert( ri != NULL ); + assert( bvout != NULL ); + + BER_BVZERO( bvout ); + +#define WHATSLEFT ( sizeof( buf ) - ( ptr - buf ) ) + + for ( i = 0; ri->ri_num[ i ] != SLAP_RETRYNUM_TAIL; i++ ) { + if ( i > 0 ) { + if ( WHATSLEFT <= 1 ) { + return 1; + } + *ptr++ = ';'; + } + + ptr += snprintf( ptr, WHATSLEFT, "%ld,", (long)ri->ri_interval[i] ); + if ( WHATSLEFT <= 0 ) { + return 1; + } + + if ( ri->ri_num[i] == SLAP_RETRYNUM_FOREVER ) { + if ( WHATSLEFT <= 1 ) { + return 1; + } + *ptr++ = '+'; + + } else { + ptr += snprintf( ptr, WHATSLEFT, "%d", ri->ri_num[i] ); + if ( WHATSLEFT <= 0 ) { + return 1; + } + } + } + + bv.bv_val = buf; + bv.bv_len = ptr - buf; + + ber_dupbv( bvout, &bv ); + + return 0; +} + +void +slap_retry_info_destroy( + slap_retry_info_t *ri ) +{ + assert( ri != NULL ); + + assert( ri->ri_interval != NULL ); + ch_free( ri->ri_interval ); + ri->ri_interval = NULL; + + assert( ri->ri_num != NULL ); + ch_free( ri->ri_num ); + ri->ri_num = NULL; +} + static int ldap_back_cf_gen( ConfigArgs *c ) { ldapinfo_t *li = ( ldapinfo_t * )c->be->be_private; - int rc; + int rc = 0; int i; if ( c->op == SLAP_CONFIG_EMIT ) { struct berval bv = BER_BVNULL; - rc = 0; if ( li == NULL ) { return 1; @@ -678,6 +833,18 @@ ldap_back_cf_gen( ConfigArgs *c ) } } break; + case LDAP_BACK_CFG_QUARANTINE: + if ( !LDAP_BACK_QUARANTINE( li ) ) { + rc = 1; + break; + } + + rc = slap_retry_info_unparse( &li->li_quarantine, &bv ); + if ( rc == 0 ) { + ber_bvarray_add( &c->rvalue_vals, &bv ); + } + break; + default: /* FIXME: we need to handle all... */ assert( 0 ); @@ -686,7 +853,6 @@ ldap_back_cf_gen( ConfigArgs *c ) return rc; } else if ( c->op == LDAP_MOD_DELETE ) { - rc = 0; switch( c->type ) { case LDAP_BACK_CFG_URI: if ( li->li_uri != NULL ) { @@ -776,6 +942,15 @@ ldap_back_cf_gen( ConfigArgs *c ) li->li_flags &= ~LDAP_BACK_F_SINGLECONN; break; + case LDAP_BACK_CFG_QUARANTINE: + if ( !LDAP_BACK_QUARANTINE( li ) ) { + break; + } + + slap_retry_info_destroy( &li->li_quarantine ); + li->li_isquarantined = 0; + break; + default: /* FIXME: we need to handle all... */ assert( 0 ); @@ -1461,6 +1636,23 @@ done_url:; li->li_flags |= mask; } break; + case LDAP_BACK_CFG_QUARANTINE: + if ( LDAP_BACK_QUARANTINE( li ) ) { + snprintf( c->msg, sizeof( c->msg ), + "quarantine already defined" ); + Debug( LDAP_DEBUG_ANY, "%s: %s.\n", c->log, c->msg, 0 ); + return 1; + } + rc = slap_retry_info_parse( c->argv[1], &li->li_quarantine, + c->msg, sizeof( c->msg ) ); + if ( rc ) { + Debug( LDAP_DEBUG_ANY, "%s: %s.\n", c->log, c->msg, 0 ); + } + /* give it a chance to retry if the pattern gets reset + * via back-config */ + li->li_isquarantined = 0; + break; + case LDAP_BACK_CFG_REWRITE: snprintf( c->msg, sizeof( c->msg ), "rewrite/remap capabilities have been moved " @@ -1476,7 +1668,7 @@ done_url:; break; } - return 0; + return rc; } int diff --git a/servers/slapd/back-ldap/extended.c b/servers/slapd/back-ldap/extended.c index 69862a6a9d..2e0974452b 100644 --- a/servers/slapd/back-ldap/extended.c +++ b/servers/slapd/back-ldap/extended.c @@ -106,6 +106,8 @@ ldap_back_exop_passwd( Operation *op, SlapReply *rs ) { + ldapinfo_t *li = (ldapinfo_t *) op->o_bd->be_private; + ldapconn_t *lc; req_pwdexop_s *qpw = &op->oq_pwdexop; LDAPMessage *res; @@ -181,10 +183,18 @@ retry: goto retry; } } + + if ( LDAP_BACK_QUARANTINE( li ) ) { + ldap_back_quarantine( op, rs, 1 ); + } + if ( text ) rs->sr_text = text; send_ldap_extended( op, rs ); /* otherwise frontend resends result */ rc = rs->sr_err = SLAPD_ABANDON; + + } else if ( LDAP_BACK_QUARANTINE( li ) ) { + ldap_back_quarantine( op, rs, 1 ); } /* these have to be freed anyway... */ @@ -210,6 +220,8 @@ ldap_back_exop_generic( Operation *op, SlapReply *rs ) { + ldapinfo_t *li = (ldapinfo_t *) op->o_bd->be_private; + ldapconn_t *lc; LDAPMessage *res; ber_int_t msgid; @@ -267,10 +279,18 @@ retry: goto retry; } } + + if ( LDAP_BACK_QUARANTINE( li ) ) { + ldap_back_quarantine( op, rs, 1 ); + } + if ( text ) rs->sr_text = text; send_ldap_extended( op, rs ); /* otherwise frontend resends result */ rc = rs->sr_err = SLAPD_ABANDON; + + } else if ( LDAP_BACK_QUARANTINE( li ) ) { + ldap_back_quarantine( op, rs, 1 ); } /* these have to be freed anyway... */ diff --git a/servers/slapd/back-ldap/init.c b/servers/slapd/back-ldap/init.c index 71e13be91c..2b96dae787 100644 --- a/servers/slapd/back-ldap/init.c +++ b/servers/slapd/back-ldap/init.c @@ -317,6 +317,9 @@ ldap_back_db_destroy( if ( li->li_conninfo.lai_tree ) { avl_free( li->li_conninfo.lai_tree, ldap_back_conn_free ); } + if ( LDAP_BACK_QUARANTINE( li ) ) { + slap_retry_info_destroy( &li->li_quarantine ); + } ldap_pvt_thread_mutex_unlock( &li->li_conninfo.lai_mutex ); ldap_pvt_thread_mutex_destroy( &li->li_conninfo.lai_mutex ); diff --git a/servers/slapd/back-ldap/proto-ldap.h b/servers/slapd/back-ldap/proto-ldap.h index 1be04ec5f6..b9c4e727b3 100644 --- a/servers/slapd/back-ldap/proto-ldap.h +++ b/servers/slapd/back-ldap/proto-ldap.h @@ -77,6 +77,18 @@ ldap_back_proxy_authz_ctrl_free( Operation *op, LDAPControl ***pctrls ); +extern void +ldap_back_quarantine( + Operation *op, + SlapReply *rs, + int dolock ); + +extern void slap_retry_info_destroy( slap_retry_info_t *ri ); +extern int slap_retry_info_parse( char *in, slap_retry_info_t *ri, + char *buf, ber_len_t buflen ); +extern int slap_retry_info_unparse( slap_retry_info_t *ri, struct berval *bvout ); + + extern int chain_initialize( void ); #ifdef LDAP_DEVEL extern int distproc_initialize( void ); diff --git a/servers/slapd/back-ldap/search.c b/servers/slapd/back-ldap/search.c index 4fa338b0d7..eed46629d3 100644 --- a/servers/slapd/back-ldap/search.c +++ b/servers/slapd/back-ldap/search.c @@ -141,6 +141,8 @@ ldap_back_search( Operation *op, SlapReply *rs ) { + ldapinfo_t *li = (ldapinfo_t *) op->o_bd->be_private; + ldapconn_t *lc; struct timeval tv; time_t stoptime = (time_t)-1; @@ -464,6 +466,10 @@ retry: } finish:; + if ( LDAP_BACK_QUARANTINE( li ) ) { + ldap_back_quarantine( op, rs, 1 ); + } + if ( rc != SLAPD_ABANDON ) { send_ldap_result( op, rs ); } diff --git a/servers/slapd/back-meta/add.c b/servers/slapd/back-meta/add.c index 5835012a87..6a58cbeb97 100644 --- a/servers/slapd/back-meta/add.c +++ b/servers/slapd/back-meta/add.c @@ -220,6 +220,10 @@ retry:; } else { send_ldap_result( op, rs ); + + if ( META_BACK_QUARANTINE( mi ) ) { + meta_back_quarantine( op, rs, candidate, 1 ); + } } cleanup:; diff --git a/servers/slapd/back-meta/back-meta.h b/servers/slapd/back-meta/back-meta.h index a8c969ba34..57778543ec 100644 --- a/servers/slapd/back-meta/back-meta.h +++ b/servers/slapd/back-meta/back-meta.h @@ -227,6 +227,9 @@ typedef struct metatarget_t { struct ldaprwmap mt_rwmap; + sig_atomic_t mt_isquarantined; + slap_retry_info_t mt_quarantine; + unsigned mt_flags; #define META_BACK_TGT_ISSET(mt,f) ( ( (mt)->mt_flags & (f) ) == (f) ) #define META_BACK_TGT_ISMASK(mt,m,f) ( ( (mt)->mt_flags & (m) ) == (f) ) @@ -276,6 +279,11 @@ typedef struct metainfo_t { ldap_avl_info_t mi_conninfo; + /* NOTE: quarantine uses the connection mutex */ + slap_retry_info_t mi_quarantine; + +#define META_BACK_QUARANTINE(mi) ( (mi)->mi_quarantine.ri_num != NULL ) + unsigned mi_flags; #define li_flags mi_flags /* uses flags as defined in */ @@ -334,12 +342,18 @@ extern int meta_back_init_one_conn( Operation *op, SlapReply *rs, - metatarget_t *mt, metaconn_t *mc, int candidate, int ispriv, ldap_back_send_t sendok ); +extern void +meta_back_quarantine( + Operation *op, + SlapReply *rs, + int candidate, + int dolock ); + extern int meta_back_single_bind( Operation *op, diff --git a/servers/slapd/back-meta/bind.c b/servers/slapd/back-meta/bind.c index 560fad0a0f..07a60d0065 100644 --- a/servers/slapd/back-meta/bind.c +++ b/servers/slapd/back-meta/bind.c @@ -442,7 +442,7 @@ retry:; } if ( mi->mi_cache.ttl != META_DNCACHE_DISABLED - && op->o_req_ndn.bv_len != 0 ) + && !BER_BVISEMPTY( &op->o_req_ndn ) ) { ( void )meta_dncache_update_entry( &mi->mi_cache, &op->o_req_ndn, candidate ); @@ -453,6 +453,10 @@ return_results:; free( mdn.bv_val ); } + if ( META_BACK_QUARANTINE( mi ) ) { + meta_back_quarantine( op, rs, candidate, 1 ); + } + return rs->sr_err; } @@ -594,7 +598,7 @@ retry:; /* mc here must be the regular mc, * reset and ready for init */ rc = meta_back_init_one_conn( op, rs, - mt, mc, candidate, + mc, candidate, LDAP_BACK_CONN_ISPRIV( mc ), LDAP_BACK_DONTSEND ); if ( rc == LDAP_SUCCESS ) { @@ -655,6 +659,10 @@ done:; } } + if ( META_BACK_QUARANTINE( mi ) ) { + meta_back_quarantine( op, rs, candidate, dolock ); + } + return rc; } @@ -981,6 +989,10 @@ meta_back_op_result( ( rmatch ? rmatch : "" ) ); } + if ( META_BACK_QUARANTINE( mi ) ) { + meta_back_quarantine( op, rs, candidate, 1 ); + } + } else { for ( i = 0; i < mi->mi_ntargets; i++ ) { metasingleconn_t *msc = &mc->mc_conns[ i ]; @@ -1051,6 +1063,10 @@ meta_back_op_result( ldap_memfree( match ); } } + + if ( META_BACK_QUARANTINE( mi ) ) { + meta_back_quarantine( op, rs, i, 1 ); + } } } diff --git a/servers/slapd/back-meta/config.c b/servers/slapd/back-meta/config.c index e93bd6c41d..80397af15b 100644 --- a/servers/slapd/back-meta/config.c +++ b/servers/slapd/back-meta/config.c @@ -171,6 +171,7 @@ meta_back_db_config( mt->mt_urllist_p = mt; mt->mt_nretries = mi->mi_nretries; + mt->mt_quarantine = mi->mi_quarantine; mt->mt_flags = mi->mi_flags; mt->mt_version = mi->mi_version; mt->mt_network_timeout = mi->mi_network_timeout; @@ -981,6 +982,49 @@ meta_back_db_config( return 1; } ber_str2bv( argv[ 1 ], 0L, 1, &mi->mi_targets[ i ]->mt_pseudorootpw ); + + /* quarantine */ + } else if ( strcasecmp( argv[ 0 ], "quarantine" ) == 0 ) { + char buf[ SLAP_TEXT_BUFLEN ] = { '\0' }; + slap_retry_info_t *ri = mi->mi_ntargets ? + &mi->mi_targets[ mi->mi_ntargets - 1 ]->mt_quarantine + : &mi->mi_quarantine; + + if ( META_BACK_QUARANTINE( mi ) ) { + Debug( LDAP_DEBUG_ANY, + "%s: line %d: quarantine already defined.\n", + fname, lineno, 0 ); + return 1; + } + + switch ( argc ) { + case 2: + break; + + case 1: + Debug( LDAP_DEBUG_ANY, + "%s: line %d: missing arg in \"quarantine \".\n", + fname, lineno, 0 ); + return 1; + + default: + Debug( LDAP_DEBUG_ANY, + "%s: line %d: extra cruft after \"quarantine \".\n", + fname, lineno, 0 ); + return 1; + } + + if ( ri != &mi->mi_quarantine ) { + ri->ri_interval = NULL; + ri->ri_num = NULL; + } + + if ( slap_retry_info_parse( argv[ 1 ], ri, buf, sizeof( buf ) ) ) { + Debug( LDAP_DEBUG_ANY, + "%s line %d: %s.\n", + fname, lineno, buf ); + return 1; + } /* dn massaging */ } else if ( strcasecmp( argv[ 0 ], "suffixmassage" ) == 0 ) { diff --git a/servers/slapd/back-meta/conn.c b/servers/slapd/back-meta/conn.c index 5364650a07..3d8cc9c36d 100644 --- a/servers/slapd/back-meta/conn.c +++ b/servers/slapd/back-meta/conn.c @@ -256,13 +256,13 @@ int meta_back_init_one_conn( Operation *op, SlapReply *rs, - metatarget_t *mt, metaconn_t *mc, int candidate, int ispriv, ldap_back_send_t sendok ) { metainfo_t *mi = ( metainfo_t * )op->o_bd->be_private; + metatarget_t *mt = mi->mi_targets[ candidate ]; metasingleconn_t *msc = &mc->mc_conns[ candidate ]; int version; dncookie dc; @@ -271,6 +271,35 @@ meta_back_init_one_conn( int is_ldaps = 0; #endif /* HAVE_TLS */ + /* if the server is quarantined, and + * - the current interval did not expire yet, or + * - no more retries should occur, + * don't return the connection */ + if ( mt->mt_isquarantined ) { + slap_retry_info_t *ri = &mt->mt_quarantine; + int dont_retry = 1; + + if ( mt->mt_isquarantined == LDAP_BACK_FQ_YES ) { + dont_retry = ( ri->ri_num[ ri->ri_idx ] == SLAP_RETRYNUM_TAIL + || slap_get_time() < ri->ri_last + ri->ri_interval[ ri->ri_idx ] ); + if ( !dont_retry ) { + Debug( LDAP_DEBUG_ANY, + "%s: meta_back_init_one_conn quarantine " + "retry block #%d try #%d.\n", + op->o_log_prefix, ri->ri_idx, ri->ri_count ); + mt->mt_isquarantined = LDAP_BACK_FQ_RETRYING; + } + } + + if ( dont_retry ) { + rs->sr_err = LDAP_UNAVAILABLE; + if ( op->o_conn && ( sendok & LDAP_BACK_SENDERR ) ) { + send_ldap_result( op, rs ); + } + return rs->sr_err; + } + } + /* * Already init'ed */ @@ -542,7 +571,7 @@ meta_back_retry( ( void )rewrite_session_delete( mt->mt_rwmap.rwm_rw, op->o_conn ); /* mc here must be the regular mc, reset and ready for init */ - rc = meta_back_init_one_conn( op, rs, mt, mc, candidate, + rc = meta_back_init_one_conn( op, rs, mc, candidate, LDAP_BACK_CONN_ISPRIV( mc ), sendok ); if ( binding ) { LDAP_BACK_CONN_BINDING_SET( msc ); @@ -592,6 +621,10 @@ meta_back_retry( } } + if ( META_BACK_QUARANTINE( mi ) ) { + meta_back_quarantine( op, rs, candidate, 0 ); + } + ldap_pvt_thread_mutex_unlock( &mi->mi_conninfo.lai_mutex ); return rc == LDAP_SUCCESS ? 1 : 0; @@ -937,15 +970,13 @@ retry_lock: } for ( i = 0; i < mi->mi_ntargets; i++ ) { - metatarget_t *mt = mi->mi_targets[ i ]; - /* * The target is activated; if needed, it is * also init'd */ candidates[ i ].sr_err = meta_back_init_one_conn( op, - rs, mt, mc, i, - LDAP_BACK_CONN_ISPRIV( &mc_curr ), sendok ); + rs, mc, i, LDAP_BACK_CONN_ISPRIV( &mc_curr ), + sendok ); if ( candidates[ i ].sr_err == LDAP_SUCCESS ) { candidates[ i ].sr_tag = META_CANDIDATE; ncandidates++; @@ -1102,7 +1133,7 @@ retry_lock2:; * also init'd. In case of error, meta_back_init_one_conn * sends the appropriate result. */ - err = meta_back_init_one_conn( op, rs, mt, mc, i, + err = meta_back_init_one_conn( op, rs, mc, i, LDAP_BACK_CONN_ISPRIV( &mc_curr ), sendok ); if ( err != LDAP_SUCCESS ) { /* @@ -1161,10 +1192,8 @@ retry_lock2:; * The target is activated; if needed, it is * also init'd */ - int lerr = meta_back_init_one_conn( op, rs, - mt, mc, i, - LDAP_BACK_CONN_ISPRIV( &mc_curr ), - sendok ); + int lerr = meta_back_init_one_conn( op, rs, mc, i, + LDAP_BACK_CONN_ISPRIV( &mc_curr ), LDAP_BACK_DONTSEND ); if ( lerr == LDAP_SUCCESS ) { candidates[ i ].sr_tag = META_CANDIDATE; candidates[ i ].sr_err = LDAP_SUCCESS; @@ -1190,6 +1219,22 @@ retry_lock2:; Debug( LDAP_DEBUG_ANY, "%s: meta_back_getconn[%d] failed: %d\n", op->o_log_prefix, i, lerr ); + if ( META_BACK_ONERR_STOP( mi ) ) { + if ( sendok & LDAP_BACK_SENDERR ) { + send_ldap_result( op, rs ); + rs->sr_text = NULL; + } + if ( new_conn ) { + meta_back_freeconn( op, mc ); + + } else { + meta_back_release_conn( op, mc ); + } + + return NULL; + } + + rs->sr_text = NULL; continue; } @@ -1330,3 +1375,67 @@ meta_back_release_conn_lock( ldap_pvt_thread_mutex_unlock( &mi->mi_conninfo.lai_mutex ); } } + +void +meta_back_quarantine( + Operation *op, + SlapReply *rs, + int candidate, + int dolock ) +{ + metainfo_t *mi = (metainfo_t *)op->o_bd->be_private; + metatarget_t *mt = mi->mi_targets[ candidate ]; + + slap_retry_info_t *ri = &mt->mt_quarantine; + + if ( dolock ) { + ldap_pvt_thread_mutex_lock( &mi->mi_conninfo.lai_mutex ); + } + + if ( rs->sr_err == LDAP_UNAVAILABLE ) { + switch ( mt->mt_isquarantined ) { + case LDAP_BACK_FQ_NO: + Debug( LDAP_DEBUG_ANY, + "%s: meta_back_quarantine enter.\n", + op->o_log_prefix, 0, 0 ); + + ri->ri_idx = 0; + ri->ri_count = 0; + break; + + case LDAP_BACK_FQ_RETRYING: + Debug( LDAP_DEBUG_ANY, + "%s: meta_back_quarantine block #%d try #%d failed.\n", + op->o_log_prefix, ri->ri_idx, ri->ri_count ); + + ++ri->ri_count; + if ( ri->ri_num[ ri->ri_idx ] != SLAP_RETRYNUM_FOREVER + && ri->ri_count == ri->ri_num[ ri->ri_idx ] ) + { + ri->ri_count = 0; + ++ri->ri_idx; + } + break; + + default: + break; + } + + mt->mt_isquarantined = LDAP_BACK_FQ_YES; + ri->ri_last = slap_get_time(); + + } else if ( mt->mt_isquarantined != LDAP_BACK_FQ_NO ) { + Debug( LDAP_DEBUG_ANY, + "%s: meta_back_quarantine exit.\n", + op->o_log_prefix, ri->ri_idx, ri->ri_count ); + + ri->ri_count = 0; + ri->ri_idx = 0; + mt->mt_isquarantined = LDAP_BACK_FQ_NO; + } + + if ( dolock ) { + ldap_pvt_thread_mutex_unlock( &mi->mi_conninfo.lai_mutex ); + } +} + diff --git a/servers/slapd/back-meta/delete.c b/servers/slapd/back-meta/delete.c index b660c76dc0..812f01d667 100644 --- a/servers/slapd/back-meta/delete.c +++ b/servers/slapd/back-meta/delete.c @@ -119,6 +119,10 @@ retry:; } else { send_ldap_result( op, rs ); + + if ( META_BACK_QUARANTINE( mi ) ) { + meta_back_quarantine( op, rs, candidate, 1 ); + } } cleanup:; diff --git a/servers/slapd/back-meta/init.c b/servers/slapd/back-meta/init.c index 5462a6fc44..914b18813d 100644 --- a/servers/slapd/back-meta/init.c +++ b/servers/slapd/back-meta/init.c @@ -267,6 +267,12 @@ meta_back_db_destroy( */ if ( mi->mi_targets != NULL ) { for ( i = 0; i < mi->mi_ntargets; i++ ) { + if ( META_BACK_QUARANTINE( mi ) + && mi->mi_targets[ i ]->mt_quarantine.ri_num != mi->mi_quarantine.ri_num ) + { + slap_retry_info_destroy( &mi->mi_targets[ i ]->mt_quarantine ); + } + target_free( mi->mi_targets[ i ] ); } @@ -287,6 +293,10 @@ meta_back_db_destroy( if ( mi->mi_candidates != NULL ) { ber_memfree_x( mi->mi_candidates, NULL ); } + + if ( META_BACK_QUARANTINE( mi ) ) { + slap_retry_info_destroy( &mi->mi_quarantine ); + } } free( be->be_private ); diff --git a/servers/slapd/back-meta/modify.c b/servers/slapd/back-meta/modify.c index 3ef81f1192..29583faa1e 100644 --- a/servers/slapd/back-meta/modify.c +++ b/servers/slapd/back-meta/modify.c @@ -231,6 +231,10 @@ cleanup:; } else { send_ldap_result( op, rs ); + + if ( META_BACK_QUARANTINE( mi ) ) { + meta_back_quarantine( op, rs, candidate, 1 ); + } } done:; diff --git a/servers/slapd/back-meta/modrdn.c b/servers/slapd/back-meta/modrdn.c index 3d94f1083e..fb739fa1c0 100644 --- a/servers/slapd/back-meta/modrdn.c +++ b/servers/slapd/back-meta/modrdn.c @@ -175,6 +175,10 @@ cleanup:; } else { send_ldap_result( op, rs ); + + if ( META_BACK_QUARANTINE( mi ) ) { + meta_back_quarantine( op, rs, candidate, 1 ); + } } done:; diff --git a/servers/slapd/back-meta/search.c b/servers/slapd/back-meta/search.c index 50519be3f2..23925647ac 100644 --- a/servers/slapd/back-meta/search.c +++ b/servers/slapd/back-meta/search.c @@ -739,8 +739,7 @@ really_bad:; */ candidates[ i ].sr_msgid = META_MSGID_IGNORE; --ncandidates; - rs->sr_err = candidates[ i ].sr_err = LDAP_OTHER; - rs->sr_text = "remote server unavailable"; + rs->sr_err = candidates[ i ].sr_err; } else if ( rc == LDAP_RES_SEARCH_ENTRY ) { if ( candidates[ i ].sr_type == REP_INTERMEDIATE ) { @@ -1258,6 +1257,10 @@ finish:; ldap_controls_free( candidates[ i ].sr_ctrls ); candidates[ i ].sr_ctrls = NULL; } + + if ( META_BACK_QUARANTINE( mi ) ) { + meta_back_quarantine( op, &candidates[ i ], i, 1 ); + } } if ( mc ) { -- 2.39.5