better async connection and partial error handling

diff --git a/servers/slapd/back-meta/back-meta.h b/servers/slapd/back-meta/back-meta.h
index ee16e6b612d6daeba27c2f5526cd6ddf3cee6d31..8b71ed25f91a01a0c7202faf6d27adc18e948519 100644 (file)
--- a/servers/slapd/back-meta/back-meta.h
+++ b/servers/slapd/back-meta/back-meta.h
@@ -194,6 +194,7 @@ struct metainfo_t;
 #define        META_NOT_CANDIDATE              ((ber_tag_t)0x0)
 #define        META_CANDIDATE                  ((ber_tag_t)0x1)
 #define        META_BINDING                    ((ber_tag_t)0x2)
+#define        META_RETRYING                   ((ber_tag_t)0x4)
 
 typedef struct metasingleconn_t {
 #define META_CND_ISSET(rs,f)           ( ( (rs)->sr_tag & (f) ) == (f) )
@@ -207,6 +208,9 @@ typedef struct metasingleconn_t {
 #define META_IS_BINDING(rs)            META_CND_ISSET( (rs), META_BINDING )
 #define META_BINDING_SET(rs)           META_CND_SET( (rs), META_BINDING )
 #define META_BINDING_CLEAR(rs)         META_CND_CLEAR( (rs), META_BINDING )
+#define META_IS_RETRYING(rs)           META_CND_ISSET( (rs), META_RETRYING )
+#define META_RETRYING_SET(rs)          META_CND_SET( (rs), META_RETRYING )
+#define META_RETRYING_CLEAR(rs)                META_CND_CLEAR( (rs), META_RETRYING )
        
        LDAP                    *msc_ld;
        time_t                  msc_time;

diff --git a/servers/slapd/back-meta/bind.c b/servers/slapd/back-meta/bind.c
index a8f8ada004bf3dc1bf08f624e15285ae41156675..c8b394a1079ada4279748e443a17174bb92c7aaa 100644 (file)
--- a/servers/slapd/back-meta/bind.c
+++ b/servers/slapd/back-meta/bind.c
@@ -407,7 +407,7 @@ retry:;
                        break;
 
                case -1:
-                       ldap_get_option( msc->msc_ld, LDAP_OPT_RESULT_CODE,
+                       ldap_get_option( msc->msc_ld, LDAP_OPT_ERROR_NUMBER,
                                &rs->sr_err );
 
                        snprintf( buf, sizeof( buf ),
@@ -493,9 +493,15 @@ meta_back_single_bind(
         * and more in case of failure ... */
        /* FIXME: should we check if at least some of the op->o_ctrls
         * can/should be passed? */
-       rs->sr_err = ldap_sasl_bind( msc->msc_ld, mdn.bv_val,
+       for (;;) {
+               rs->sr_err = ldap_sasl_bind( msc->msc_ld, mdn.bv_val,
                        LDAP_SASL_SIMPLE, &op->orb_cred,
                        op->o_ctrls, NULL, &msgid );
+               if ( rs->sr_err != LDAP_X_CONNECTING ) {
+                       break;
+               }
+               ldap_pvt_thread_yield();
+       }
        meta_back_bind_op_result( op, rs, mc, candidate, msgid, LDAP_BACK_DONTSEND );
        if ( rs->sr_err != LDAP_SUCCESS ) {
                goto return_results;
@@ -564,7 +570,6 @@ meta_back_single_dobind(
        metatarget_t            *mt = mi->mi_targets[ candidate ];
        metaconn_t              *mc = *mcp;
        metasingleconn_t        *msc = &mc->mc_conns[ candidate ];
-       int                     rc;
        static struct berval    cred = BER_BVC( "" );
        int                     msgid;
 
@@ -579,19 +584,25 @@ meta_back_single_dobind(
                        ( mt->mt_idassert_flags & LDAP_BACK_AUTH_OVERRIDE ) ) )
        {
                (void)meta_back_proxy_authz_bind( mc, candidate, op, rs, sendok );
-               rc = rs->sr_err;
-               goto done;
-       }
 
-       /* FIXME: should we check if at least some of the op->o_ctrls
-        * can/should be passed? */
-       rs->sr_err = ldap_sasl_bind( msc->msc_ld, "", LDAP_SASL_SIMPLE, &cred,
-                       NULL, NULL, &msgid );
-       rc = meta_back_bind_op_result( op, rs, mc, candidate, msgid, sendok );
+       } else {
 
-done:;
-       rs->sr_err = rc;
-       if ( rc != LDAP_SUCCESS ) {
+               /* FIXME: should we check if at least some of the op->o_ctrls
+                * can/should be passed? */
+               for (;;) {
+                       rs->sr_err = ldap_sasl_bind( msc->msc_ld,
+                               "", LDAP_SASL_SIMPLE, &cred,
+                               NULL, NULL, &msgid );
+                       if ( rs->sr_err != LDAP_X_CONNECTING ) {
+                               break;
+                       }
+                       ldap_pvt_thread_yield();
+               }
+
+               rs->sr_err = meta_back_bind_op_result( op, rs, mc, candidate, msgid, sendok );
+       }
+
+       if ( rs->sr_err != LDAP_SUCCESS ) {
                if ( dolock ) {
                        ldap_pvt_thread_mutex_lock( &mi->mi_conninfo.lai_mutex );
                }
@@ -614,7 +625,7 @@ done:;
                meta_back_quarantine( op, rs, candidate );
        }
 
-       return rc;
+       return rs->sr_err;
 }
 
 /*
@@ -1457,9 +1468,15 @@ meta_back_proxy_authz_bind( metaconn_t *mc, int candidate, Operation *op, SlapRe
                switch ( method ) {
                case LDAP_AUTH_NONE:
                case LDAP_AUTH_SIMPLE:
-                       rs->sr_err = ldap_sasl_bind( msc->msc_ld,
+                       for (;;) {
+                               rs->sr_err = ldap_sasl_bind( msc->msc_ld,
                                        binddn.bv_val, LDAP_SASL_SIMPLE,
                                        &cred, NULL, NULL, &msgid );
+                               if ( rs->sr_err != LDAP_X_CONNECTING ) {
+                                       break;
+                               }
+                               ldap_pvt_thread_yield();
+                       }
                        rc = meta_back_bind_op_result( op, rs, mc, candidate, msgid, sendok );
                        if ( rc == LDAP_SUCCESS ) {
                                /* set rebind stuff in case of successful proxyAuthz bind,

diff --git a/servers/slapd/back-meta/conn.c b/servers/slapd/back-meta/conn.c
index 344b075a50baa3d297a7950ea05b0fbf27cfb9aa..91927ba33a037a365832787f29437ecda41ea1a4 100644 (file)
--- a/servers/slapd/back-meta/conn.c
+++ b/servers/slapd/back-meta/conn.c
@@ -644,7 +644,8 @@ meta_back_retry(
        metaconn_t              *mc = *mcp;
        metasingleconn_t        *msc = &mc->mc_conns[ candidate ];
        int                     rc = LDAP_UNAVAILABLE,
-                               binding;
+                               binding,
+                               quarantine = 1;
 
        ldap_pvt_thread_mutex_lock( &mi->mi_conninfo.lai_mutex );
 
@@ -687,6 +688,7 @@ meta_back_retry(
                }
 
                if ( rc == LDAP_SUCCESS ) {
+                       quarantine = 0;
                        rc = meta_back_single_dobind( op, rs, mcp, candidate,
                                sendok, mt->mt_nretries, 0 );
 
@@ -776,7 +778,7 @@ meta_back_retry(
                }
        }
 
-       if ( META_BACK_TGT_QUARANTINE( mt ) ) {
+       if ( quarantine && META_BACK_TGT_QUARANTINE( mt ) ) {
                meta_back_quarantine( op, rs, candidate );
        }
 
@@ -1451,9 +1453,9 @@ retry_lock2:;
                                int lerr = meta_back_init_one_conn( op, rs, mc, i,
                                        LDAP_BACK_CONN_ISPRIV( &mc_curr ),
                                        LDAP_BACK_DONTSEND, !new_conn );
+                               candidates[ i ].sr_err = lerr;
                                if ( lerr == LDAP_SUCCESS ) {
                                        META_CANDIDATE_SET( &candidates[ i ] );
-                                       candidates[ i ].sr_err = LDAP_SUCCESS;
                                        ncandidates++;
 
                                        Debug( LDAP_DEBUG_TRACE, "%s: meta_back_getconn[%d]\n",
@@ -1461,7 +1463,6 @@ retry_lock2:;
 
                                } else if ( lerr == LDAP_UNAVAILABLE && !META_BACK_ONERR_STOP( mi ) ) {
                                        META_CANDIDATE_SET( &candidates[ i ] );
-                                       candidates[ i ].sr_err = LDAP_UNAVAILABLE;
 
                                        Debug( LDAP_DEBUG_TRACE, "%s: meta_back_getconn[%d] %s\n",
                                                op->o_log_prefix, i,
@@ -1478,7 +1479,6 @@ retry_lock2:;
                                                ( void )meta_clear_one_candidate( op, mc, i );
                                        }
                                        /* leave the target candidate, but record the error for later use */
-                                       candidates[ i ].sr_err = lerr;
                                        err = lerr;
 
                                        if ( lerr == LDAP_UNAVAILABLE && mt->mt_isquarantined != LDAP_BACK_FQ_NO ) {

diff --git a/servers/slapd/back-meta/search.c b/servers/slapd/back-meta/search.c
index 05d41ed46d87a0998ac061ea1f0119af190ba552..94aafeead06917367fde857e95eefb4b06279196 100644 (file)
--- a/servers/slapd/back-meta/search.c
+++ b/servers/slapd/back-meta/search.c
@@ -229,6 +229,10 @@ meta_search_dobind_init(
 
        assert( msc->msc_ld != NULL );
 
+       /* connect must be async */
+retry:;
+       ldap_set_option( msc->msc_ld, LDAP_OPT_CONNECT_ASYNC, LDAP_OPT_ON );
+
        rc = ldap_sasl_bind( msc->msc_ld, binddn.bv_val, LDAP_SASL_SIMPLE, &cred,
                        NULL, NULL, &candidates[ candidate ].sr_msgid );
 
@@ -249,13 +253,55 @@ meta_search_dobind_init(
                META_BINDING_SET( &candidates[ candidate ] );
                return META_SEARCH_BINDING;
 
+       case LDAP_X_CONNECTING:
+               /* must retry, same conn */
+               candidates[ candidate ].sr_msgid = META_MSGID_NEED_BIND;
+               ldap_pvt_thread_mutex_lock( &mi->mi_conninfo.lai_mutex );
+               LDAP_BACK_CONN_BINDING_CLEAR( msc );
+               ldap_pvt_thread_mutex_unlock( &mi->mi_conninfo.lai_mutex );
+               return META_SEARCH_NEED_BIND;
+
        case LDAP_SERVER_DOWN:
 down:;
                /* This is the worst thing that could happen:
                 * the search will wait until the retry is over. */
-               if ( meta_back_retry( op, rs, mcp, candidate, LDAP_BACK_DONTSEND ) ) {
-                       candidates[ candidate ].sr_msgid = META_MSGID_IGNORE;
-                       return META_SEARCH_CANDIDATE;
+               if ( !META_IS_RETRYING( &candidates[ candidate ] ) ) {
+                       META_RETRYING_SET( &candidates[ candidate ] );
+
+                       ldap_pvt_thread_mutex_lock( &mi->mi_conninfo.lai_mutex );
+
+                       assert( mc->mc_refcnt > 0 );
+                       if ( LogTest( LDAP_DEBUG_ANY ) ) {
+                               char    buf[ SLAP_TEXT_BUFLEN ];
+
+                               /* this lock is required; however,
+                                * it's invoked only when logging is on */
+                               ldap_pvt_thread_mutex_lock( &mt->mt_uri_mutex );
+                               snprintf( buf, sizeof( buf ),
+                                       "retrying URI=\"%s\" DN=\"%s\"",
+                                       mt->mt_uri,
+                                       BER_BVISNULL( &msc->msc_bound_ndn ) ?
+                                               "" : msc->msc_bound_ndn.bv_val );
+                               ldap_pvt_thread_mutex_unlock( &mt->mt_uri_mutex );
+
+                               Debug( LDAP_DEBUG_ANY,
+                                       "%s meta_search_dobind_init[%d]: %s.\n",
+                                       op->o_log_prefix, candidate, buf );
+                       }
+
+                       meta_clear_one_candidate( op, mc, candidate );
+                       LDAP_BACK_CONN_ISBOUND_CLEAR( msc );
+
+                       ( void )rewrite_session_delete( mt->mt_rwmap.rwm_rw, op->o_conn );
+
+                       /* mc here must be the regular mc, reset and ready for init */
+                       rc = meta_back_init_one_conn( op, rs, mc, candidate,
+                               LDAP_BACK_CONN_ISPRIV( mc ), LDAP_BACK_DONTSEND, 0 );
+
+                       LDAP_BACK_CONN_BINDING_SET( msc );
+                       ldap_pvt_thread_mutex_unlock( &mi->mi_conninfo.lai_mutex );
+
+                       goto retry;
                }
 
                if ( *mcp == NULL ) {
@@ -272,6 +318,7 @@ other:;
 
                ldap_pvt_thread_mutex_lock( &mi->mi_conninfo.lai_mutex );
                meta_clear_one_candidate( op, mc, candidate );
+               candidates[ candidate ].sr_err = rc;
                if ( META_BACK_ONERR_STOP( mi ) ) {
                        LDAP_BACK_CONN_TAINTED_SET( mc );
                        meta_back_release_conn_lock( op, mc, 0 );
@@ -280,10 +327,6 @@ other:;
                        retcode = META_SEARCH_ERR;
 
                } else {
-                       if ( META_BACK_ONERR_REPORT( mi ) ) {
-                               candidates[ candidate ].sr_err = rc;
-                       }
-
                        retcode = META_SEARCH_NOT_CANDIDATE;
                }
                candidates[ candidate ].sr_msgid = META_MSGID_IGNORE;
@@ -326,16 +369,14 @@ meta_search_dobind_result(
        ldap_pvt_thread_mutex_lock( &mi->mi_conninfo.lai_mutex );
        LDAP_BACK_CONN_BINDING_CLEAR( msc );
        if ( rc != LDAP_SUCCESS ) {
+               meta_clear_one_candidate( op, mc, candidate );
+               candidates[ candidate ].sr_err = rc;
                if ( META_BACK_ONERR_STOP( mi ) ) {
                        LDAP_BACK_CONN_TAINTED_SET( mc );
-                       meta_clear_one_candidate( op, mc, candidate );
                        meta_back_release_conn_lock( op, mc, 0 );
                        *mcp = NULL;
                        retcode = META_SEARCH_ERR;
                        rs->sr_err = rc;
-
-               } else if ( META_BACK_ONERR_REPORT( mi ) ) {
-                       candidates[ candidate ].sr_err = rc;
                }
 
        } else {
@@ -349,6 +390,9 @@ meta_search_dobind_result(
                        LDAP_BACK_CONN_ISBOUND_SET( msc );
                }
                retcode = META_SEARCH_CANDIDATE;
+
+               /* connect must be async */
+               ldap_set_option( msc->msc_ld, LDAP_OPT_CONNECT_ASYNC, LDAP_OPT_OFF );
        }
 
        candidates[ candidate ].sr_msgid = META_MSGID_IGNORE;
@@ -388,12 +432,10 @@ meta_back_search_start(
                        "%s: meta_back_search_start candidate=%d ld=NULL%s.\n",
                        op->o_log_prefix, candidate,
                        META_BACK_ONERR_STOP( mi ) ? "" : " (ignored)" );
+               candidates[ candidate ].sr_err = LDAP_OTHER;
                if ( META_BACK_ONERR_STOP( mi ) ) {
                        return META_SEARCH_ERR;
                }
-               if ( META_BACK_ONERR_REPORT( mi ) ) {
-                       candidates[ candidate ].sr_err = LDAP_OTHER;
-               }
                candidates[ candidate ].sr_msgid = META_MSGID_IGNORE;
                return META_SEARCH_NOT_CANDIDATE;
        }
@@ -846,6 +888,7 @@ getconn:;
                                        break;
 
                                case META_SEARCH_ERR:
+                                       candidates[ i ].sr_err = rs->sr_err;
                                        if ( META_BACK_ONERR_STOP( mi ) ) {
                                                savepriv = op->o_private;
                                                op->o_private = (void *)i;
@@ -853,9 +896,6 @@ getconn:;
                                                op->o_private = savepriv;
                                                goto finish;
                                        }
-                                       if ( META_BACK_ONERR_REPORT( mi ) ) {
-                                               candidates[ i ].sr_err = rs->sr_err;
-                                       }
                                        /* fallthru */
 
                                case META_SEARCH_NOT_CANDIDATE:
@@ -877,6 +917,7 @@ getconn:;
                                                break;
 
                                        case META_SEARCH_ERR:
+                                               candidates[ i ].sr_err = rs->sr_err;
                                                if ( META_BACK_ONERR_STOP( mi ) ) {
                                                        savepriv = op->o_private;
                                                        op->o_private = (void *)i;
@@ -884,9 +925,6 @@ getconn:;
                                                        op->o_private = savepriv;
                                                        goto finish;
                                                }
-                                               if ( META_BACK_ONERR_REPORT( mi ) ) {
-                                                       candidates[ i ].sr_err = rs->sr_err;
-                                               }
                                                /* fallthru */
 
                                        case META_SEARCH_NOT_CANDIDATE:
@@ -970,6 +1008,7 @@ really_bad:;
                                                        candidates[ i ].sr_msgid = META_MSGID_IGNORE;
                                                        --ncandidates;
 
+                                                       candidates[ i ].sr_err = rs->sr_err;
                                                        if ( META_BACK_ONERR_STOP( mi ) ) {
                                                                savepriv = op->o_private;
                                                                op->o_private = (void *)i;
@@ -977,9 +1016,6 @@ really_bad:;
                                                                op->o_private = savepriv;
                                                                goto finish;
                                                        }
-                                                       if ( META_BACK_ONERR_REPORT( mi ) ) {
-                                                               candidates[ i ].sr_err = rs->sr_err;
-                                                       }
                                                        break;
 
                                                case META_SEARCH_BINDING:
@@ -994,6 +1030,7 @@ really_bad:;
                                                }
                                        }
 
+                                       candidates[ i ].sr_err = rs->sr_err;
                                        if ( META_BACK_ONERR_STOP( mi ) ) {
                                                savepriv = op->o_private;
                                                op->o_private = (void *)i;
@@ -1001,9 +1038,6 @@ really_bad:;
                                                op->o_private = savepriv;
                                                goto finish;
                                        }
-                                       if ( META_BACK_ONERR_REPORT( mi ) ) {
-                                               candidates[ i ].sr_err = rs->sr_err;
-                                       }
                                }
 
                                /*
@@ -1270,6 +1304,7 @@ really_bad:;
                                                 * the target enforced a limit lower
                                                 * than what requested by the proxy;
                                                 * ignore it */
+                                               candidates[ i ].sr_err = rs->sr_err;
                                                if ( rs->sr_nentries == op->ors_slimit
                                                        || META_BACK_ONERR_STOP( mi ) )
                                                {
@@ -1281,12 +1316,10 @@ really_bad:;
                                                        res = NULL;
                                                        goto finish;
                                                }
-                                               if ( META_BACK_ONERR_REPORT( mi ) ) {
-                                                       candidates[ i ].sr_err = rs->sr_err;
-                                               }
                                                break;
        
                                        default:
+                                               candidates[ i ].sr_err = rs->sr_err;
                                                if ( META_BACK_ONERR_STOP( mi ) ) {
                                                        savepriv = op->o_private;
                                                        op->o_private = (void *)i;
@@ -1296,9 +1329,6 @@ really_bad:;
                                                        res = NULL;
                                                        goto finish;
                                                }
-                                               if ( META_BACK_ONERR_REPORT( mi ) ) {
-                                                       candidates[ i ].sr_err = rs->sr_err;
-                                               }
                                                break;
                                        }
        
@@ -1331,6 +1361,7 @@ really_bad:;
                                                candidates[ i ].sr_msgid = META_MSGID_IGNORE;
                                                --ncandidates;
        
+                                               candidates[ i ].sr_err = rs->sr_err;
                                                if ( META_BACK_ONERR_STOP( mi ) ) {
                                                        savepriv = op->o_private;
                                                        op->o_private = (void *)i;
@@ -1340,9 +1371,6 @@ really_bad:;
                                                        res = NULL;
                                                        goto finish;
                                                }
-                                               if ( META_BACK_ONERR_REPORT( mi ) ) {
-                                                       candidates[ i ].sr_err = rs->sr_err;
-                                               }
                                                break;
        
                                        default: