git.sur5r.net Git - openldap/blob - servers/slapd/back-asyncmeta/conn.c

   1 /* conn.c - handles connections to remote targets */
   2 /* $OpenLDAP$ */
   3 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
   4  *
   5  * Copyright 2016-2017 The OpenLDAP Foundation.
   6  * Portions Copyright 2016 Symas Corporation.
   7  * All rights reserved.
   8  *
   9  * Redistribution and use in source and binary forms, with or without
  10  * modification, are permitted only as authorized by the OpenLDAP
  11  * Public License.
  12  *
  13  * A copy of this license is available in the file LICENSE in the
  14  * top-level directory of the distribution or, alternatively, at
  15  * <http://www.OpenLDAP.org/license.html>.
  16  */
  17
  18 /* ACKNOWLEDGEMENTS:
  19 + * This work was developed by Symas Corporation
  20 + * based on back-meta module for inclusion in OpenLDAP Software.
  21 + * This work was sponsored by Ericsson. */
  22
  23 #include "portable.h"
  24
  25 #include <stdio.h>
  26
  27 #include <ac/errno.h>
  28 #include <ac/socket.h>
  29 #include <ac/string.h>
  30
  31
  32 #define AVL_INTERNAL
  33 #include "slap.h"
  34 #include "../back-ldap/back-ldap.h"
  35 #include "back-asyncmeta.h"
  36
  37
  38 /*
  39  * Debug stuff (got it from libavl)
  40  */
  41 #if META_BACK_PRINT_CONNTREE > 0
  42
  43 static void
  44 asyncmeta_back_ravl_print( Avlnode *root, int depth )
  45 {
  46         int             i;
  47
  48         if ( root == 0 ) {
  49                 return;
  50         }
  51
  52         asyncmeta_back_ravl_print( root->avl_right, depth + 1 );
  53
  54         for ( i = 0; i < depth; i++ ) {
  55                 fprintf( stderr, "-" );
  56         }
  57         fputc( ' ', stderr );
  58
  59         asyncmeta_back_print( (a_metaconn_t *)root->avl_data,
  60                 avl_bf2str( root->avl_bf ) );
  61
  62         asyncmeta_back_ravl_print( root->avl_left, depth + 1 );
  63 }
  64
  65 /* NOTE: duplicate from back-ldap/bind.c */
  66 static char* priv2str[] = {
  67         "privileged",
  68         "privileged/TLS",
  69         "anonymous",
  70         "anonymous/TLS",
  71         "bind",
  72         "bind/TLS",
  73         NULL
  74 };
  75
  76 #endif /* META_BACK_PRINT_CONNTREE */
  77 /*
  78  * End of debug stuff
  79  */
  80
  81 /*
  82  * asyncmeta_conn_alloc
  83  *
  84  * Allocates a connection structure, making room for all the referenced targets
  85  */
  86 static a_metaconn_t *
  87 asyncmeta_conn_alloc(
  88         a_metainfo_t    *mi)
  89 {
  90         a_metaconn_t    *mc;
  91         int             ntargets = mi->mi_ntargets;
  92
  93         assert( ntargets > 0 );
  94
  95         /* malloc all in one */
  96         mc = ( a_metaconn_t * )ch_calloc( 1, sizeof( a_metaconn_t ) + ntargets * sizeof( a_metasingleconn_t ));
  97         if ( mc == NULL ) {
  98                 return NULL;
  99         }
 100
 101         mc->mc_info = mi;
 102         ldap_pvt_thread_mutex_init( &mc->mc_om_mutex);
 103         mc->mc_authz_target = META_BOUND_NONE;
 104         mc->mc_conns = (a_metasingleconn_t *)(mc+1);
 105         return mc;
 106 }
 107
 108 /*
 109  * asyncmeta_init_one_conn
 110  *
 111  * Initializes one connection
 112  */
 113 int
 114 asyncmeta_init_one_conn(
 115         Operation               *op,
 116         SlapReply               *rs,
 117         a_metaconn_t            *mc,
 118         int                     candidate,
 119         int                     ispriv,
 120         ldap_back_send_t        sendok,
 121         int                     dolock)
 122 {
 123         a_metainfo_t            *mi = mc->mc_info;
 124         a_metatarget_t          *mt = mi->mi_targets[ candidate ];
 125         a_metasingleconn_t      *msc = NULL;
 126         int                     version;
 127         a_dncookie              dc;
 128         int                     isauthz = ( candidate == mc->mc_authz_target );
 129         int                     do_return = 0;
 130         int                     nretries = 2;
 131
 132 #ifdef HAVE_TLS
 133         int                     is_ldaps = 0;
 134         int                     do_start_tls = 0;
 135 #endif /* HAVE_TLS */
 136
 137         /* if the server is quarantined, and
 138          * - the current interval did not expire yet, or
 139          * - no more retries should occur,
 140          * don't return the connection */
 141         if ( mt->mt_isquarantined ) {
 142                 slap_retry_info_t       *ri = &mt->mt_quarantine;
 143                 int                     dont_retry = 0;
 144
 145                 if ( mt->mt_quarantine.ri_interval ) {
 146                         ldap_pvt_thread_mutex_lock( &mt->mt_quarantine_mutex );
 147                         dont_retry = ( mt->mt_isquarantined > LDAP_BACK_FQ_NO );
 148                         if ( dont_retry ) {
 149                                 dont_retry = ( ri->ri_num[ ri->ri_idx ] == SLAP_RETRYNUM_TAIL
 150                                         || slap_get_time() < ri->ri_last + ri->ri_interval[ ri->ri_idx ] );
 151                                 if ( !dont_retry ) {
 152                                         if ( LogTest( LDAP_DEBUG_ANY ) ) {
 153                                                 char    buf[ SLAP_TEXT_BUFLEN ];
 154
 155                                                 snprintf( buf, sizeof( buf ),
 156                                                         "asyncmeta_init_one_conn[%d]: quarantine "
 157                                                         "retry block #%d try #%d",
 158                                                         candidate, ri->ri_idx, ri->ri_count );
 159                                                 Debug( LDAP_DEBUG_ANY, "%s %s.\n",
 160                                                         op->o_log_prefix, buf, 0 );
 161                                         }
 162
 163                                         mt->mt_isquarantined = LDAP_BACK_FQ_RETRYING;
 164                                 }
 165
 166                         }
 167                         ldap_pvt_thread_mutex_unlock( &mt->mt_quarantine_mutex );
 168                 }
 169
 170                 if ( dont_retry ) {
 171                         rs->sr_err = LDAP_UNAVAILABLE;
 172                         if ( op->o_conn && ( sendok & LDAP_BACK_SENDERR ) ) {
 173                                 rs->sr_text = "Target is quarantined";
 174                                         send_ldap_result( op, rs );
 175                         }
 176                         return rs->sr_err;
 177                 }
 178         }
 179                 msc = &mc->mc_conns[candidate];
 180 retry_lock:;
 181         /*
 182          * Already init'ed
 183          */
 184         if ( LDAP_BACK_CONN_ISBOUND( msc )
 185                 || LDAP_BACK_CONN_ISANON( msc ) )
 186         {
 187                 assert( msc->msc_ld != NULL );
 188                 rs->sr_err = LDAP_SUCCESS;
 189                 do_return = 1;
 190
 191         } else if ( META_BACK_CONN_CREATING( msc )
 192                 || LDAP_BACK_CONN_BINDING( msc ) )
 193         {
 194                 /* sounds more appropriate */
 195                 if (nretries >= 0) {
 196                         nretries--;
 197                         ldap_pvt_thread_yield();
 198                         goto retry_lock;
 199                 }
 200                 rs->sr_err = LDAP_UNAVAILABLE;
 201                 do_return = 1;
 202
 203         } else if ( META_BACK_CONN_INITED( msc ) ) {
 204                 assert( msc->msc_ld != NULL );
 205                 rs->sr_err = LDAP_SUCCESS;
 206                 do_return = 1;
 207
 208         } else {
 209                 /*
 210                  * creating...
 211                  */
 212                 META_BACK_CONN_CREATING_SET( msc );
 213         }
 214
 215         if ( do_return ) {
 216                 if ( rs->sr_err != LDAP_SUCCESS
 217                         && op->o_conn
 218                         && ( sendok & LDAP_BACK_SENDERR ) )
 219                 {
 220                         send_ldap_result( op, rs );
 221                 }
 222
 223                 return rs->sr_err;
 224         }
 225
 226         assert( msc->msc_ld == NULL );
 227
 228         /*
 229          * Attempts to initialize the connection to the target ds
 230          */
 231         ldap_pvt_thread_mutex_lock( &mt->mt_uri_mutex );
 232
 233         rs->sr_err = ldap_initialize( &msc->msc_ld, mt->mt_uri );
 234 #ifdef HAVE_TLS
 235         is_ldaps = ldap_is_ldaps_url( mt->mt_uri );
 236 #endif /* HAVE_TLS */
 237         ldap_pvt_thread_mutex_unlock( &mt->mt_uri_mutex );
 238         if ( rs->sr_err != LDAP_SUCCESS ) {
 239                 goto error_return;
 240         }
 241         msc->msc_ldr = ldap_dup(msc->msc_ld);
 242         if (!msc->msc_ldr) {
 243                 ldap_ld_free(msc->msc_ld, 0, NULL, NULL);
 244                 rs->sr_err = LDAP_NO_MEMORY;
 245                 goto error_return;
 246         }
 247
 248         /*
 249          * Set LDAP version. This will always succeed: If the client
 250          * bound with a particular version, then so can we.
 251          */
 252         if ( mt->mt_version != 0 ) {
 253                 version = mt->mt_version;
 254
 255         } else if ( op->o_conn->c_protocol != 0 ) {
 256                 version = op->o_conn->c_protocol;
 257
 258         } else {
 259                 version = LDAP_VERSION3;
 260         }
 261         ldap_set_option( msc->msc_ld, LDAP_OPT_PROTOCOL_VERSION, &version );
 262         ldap_set_urllist_proc( msc->msc_ld, mt->mt_urllist_f, mt->mt_urllist_p );
 263
 264         /* automatically chase referrals ("chase-referrals [{yes|no}]" statement) */
 265         ldap_set_option( msc->msc_ld, LDAP_OPT_REFERRALS,
 266                 META_BACK_TGT_CHASE_REFERRALS( mt ) ? LDAP_OPT_ON : LDAP_OPT_OFF );
 267
 268         slap_client_keepalive(msc->msc_ld, &mt->mt_tls.sb_keepalive);
 269
 270 #ifdef HAVE_TLS
 271         {
 272                 slap_bindconf *sb = NULL;
 273
 274                 if ( ispriv ) {
 275                         sb = &mt->mt_idassert.si_bc;
 276                 } else {
 277                         sb = &mt->mt_tls;
 278                 }
 279
 280                 if ( sb->sb_tls_do_init ) {
 281                         bindconf_tls_set( sb, msc->msc_ld );
 282                 } else if ( sb->sb_tls_ctx ) {
 283                         ldap_set_option( msc->msc_ld, LDAP_OPT_X_TLS_CTX, sb->sb_tls_ctx );
 284                 }
 285
 286                 if ( !is_ldaps ) {
 287                         if ( sb == &mt->mt_idassert.si_bc && sb->sb_tls_ctx ) {
 288                                 do_start_tls = 1;
 289
 290                         } else if ( META_BACK_TGT_USE_TLS( mt )
 291                                 || ( op->o_conn->c_is_tls && META_BACK_TGT_PROPAGATE_TLS( mt ) ) )
 292                         {
 293                                 do_start_tls = 1;
 294                         }
 295                 }
 296         }
 297
 298         /* start TLS ("tls [try-]{start|propagate}" statement) */
 299         if ( do_start_tls ) {
 300 #ifdef SLAP_STARTTLS_ASYNCHRONOUS
 301                 /*
 302                  * use asynchronous StartTLS; in case, chase referral
 303                  * FIXME: OpenLDAP does not return referral on StartTLS yet
 304                  */
 305                 int             msgid;
 306
 307                 rs->sr_err = ldap_start_tls( msc->msc_ld, NULL, NULL, &msgid );
 308                 if ( rs->sr_err == LDAP_SUCCESS ) {
 309                         LDAPMessage     *res = NULL;
 310                         int             rc, nretries = mt->mt_nretries;
 311                         struct timeval  tv;
 312
 313                         LDAP_BACK_TV_SET( &tv );
 314
 315 retry:;
 316                         rc = ldap_result( msc->msc_ld, msgid, LDAP_MSG_ALL, &tv, &res );
 317                         switch ( rc ) {
 318                         case -1:
 319                                 rs->sr_err = LDAP_OTHER;
 320                                 break;
 321
 322                         case 0:
 323                                 if ( nretries != 0 ) {
 324                                         if ( nretries > 0 ) {
 325                                                 nretries--;
 326                                         }
 327                                         LDAP_BACK_TV_SET( &tv );
 328                                         goto retry;
 329                                 }
 330                                 rs->sr_err = LDAP_OTHER;
 331                                 break;
 332
 333                         default:
 334                                 /* only touch when activity actually took place... */
 335                                 if ( mi->mi_idle_timeout != 0 && msc->msc_time < op->o_time ) {
 336                                         msc->msc_time = op->o_time;
 337                                 }
 338                                 break;
 339                         }
 340
 341                         if ( rc == LDAP_RES_EXTENDED ) {
 342                                 struct berval   *data = NULL;
 343
 344                                 /* NOTE: right now, data is unused, so don't get it */
 345                                 rs->sr_err = ldap_parse_extended_result( msc->msc_ld,
 346                                         res, NULL, NULL /* &data */ , 0 );
 347                                 if ( rs->sr_err == LDAP_SUCCESS ) {
 348                                         int             err;
 349
 350                                         /* FIXME: matched? referrals? response controls? */
 351                                         rs->sr_err = ldap_parse_result( msc->msc_ld,
 352                                                 res, &err, NULL, NULL, NULL, NULL, 1 );
 353                                         res = NULL;
 354
 355                                         if ( rs->sr_err == LDAP_SUCCESS ) {
 356                                                 rs->sr_err = err;
 357                                         }
 358                                         rs->sr_err = slap_map_api2result( rs );
 359
 360                                         /* FIXME: in case a referral
 361                                          * is returned, should we try
 362                                          * using it instead of the
 363                                          * configured URI? */
 364                                         if ( rs->sr_err == LDAP_SUCCESS ) {
 365                                                 ldap_install_tls( msc->msc_ld );
 366
 367                                         } else if ( rs->sr_err == LDAP_REFERRAL ) {
 368                                                 /* FIXME: LDAP_OPERATIONS_ERROR? */
 369                                                 rs->sr_err = LDAP_OTHER;
 370                                                 rs->sr_text = "Unwilling to chase referral "
 371                                                         "returned by Start TLS exop";
 372                                         }
 373
 374                                         if ( data ) {
 375                                                 ber_bvfree( data );
 376                                         }
 377                                 }
 378
 379                         } else {
 380                                 rs->sr_err = LDAP_OTHER;
 381                         }
 382
 383                         if ( res != NULL ) {
 384                                 ldap_msgfree( res );
 385                         }
 386                 }
 387 #else /* ! SLAP_STARTTLS_ASYNCHRONOUS */
 388                 /*
 389                  * use synchronous StartTLS
 390                  */
 391                 rs->sr_err = ldap_start_tls_s( msc->msc_ld, NULL, NULL );
 392 #endif /* ! SLAP_STARTTLS_ASYNCHRONOUS */
 393
 394                 /* if StartTLS is requested, only attempt it if the URL
 395                  * is not "ldaps://"; this may occur not only in case
 396                  * of misconfiguration, but also when used in the chain
 397                  * overlay, where the "uri" can be parsed out of a referral */
 398                 if ( rs->sr_err == LDAP_SERVER_DOWN
 399                         || ( rs->sr_err != LDAP_SUCCESS
 400                                 && META_BACK_TGT_TLS_CRITICAL( mt ) ) )
 401                 {
 402
 403 #ifdef DEBUG_205
 404                         Debug( LDAP_DEBUG_ANY,
 405                                 "### %s asyncmeta_init_one_conn(TLS) "
 406                                 "ldap_unbind_ext[%d] ld=%p\n",
 407                                 op->o_log_prefix, candidate,
 408                                 (void *)msc->msc_ld );
 409 #endif /* DEBUG_205 */
 410
 411                         goto error_return;
 412                 }
 413         }
 414 #endif /* HAVE_TLS */
 415
 416         /*
 417          * Set the network timeout if set
 418          */
 419         if ( mt->mt_network_timeout != 0 ) {
 420                 struct timeval  network_timeout;
 421
 422                 network_timeout.tv_usec = 0;
 423                 network_timeout.tv_sec = mt->mt_network_timeout;
 424
 425                 ldap_set_option( msc->msc_ld, LDAP_OPT_NETWORK_TIMEOUT,
 426                                 (void *)&network_timeout );
 427         }
 428
 429         /*
 430          * If the connection DN is not null, an attempt to rewrite it is made
 431          */
 432
 433         if ( ispriv ) {
 434                 if ( !BER_BVISNULL( &mt->mt_idassert_authcDN ) ) {
 435                         ber_bvreplace( &msc->msc_bound_ndn, &mt->mt_idassert_authcDN );
 436                         if ( !BER_BVISNULL( &mt->mt_idassert_passwd ) ) {
 437                                 if ( !BER_BVISNULL( &msc->msc_cred ) ) {
 438                                         memset( msc->msc_cred.bv_val, 0,
 439                                                 msc->msc_cred.bv_len );
 440                                 }
 441                                 ber_bvreplace( &msc->msc_cred, &mt->mt_idassert_passwd );
 442                         }
 443                         LDAP_BACK_CONN_ISIDASSERT_SET( msc );
 444
 445                 } else {
 446                         ber_bvreplace( &msc->msc_bound_ndn, &slap_empty_bv );
 447                 }
 448
 449         } else {
 450                 if ( !BER_BVISNULL( &msc->msc_cred ) ) {
 451                         memset( msc->msc_cred.bv_val, 0, msc->msc_cred.bv_len );
 452                         ber_memfree_x( msc->msc_cred.bv_val, NULL );
 453                         BER_BVZERO( &msc->msc_cred );
 454                 }
 455                 if ( !BER_BVISNULL( &msc->msc_bound_ndn ) ) {
 456                         ber_memfree_x( msc->msc_bound_ndn.bv_val, NULL );
 457                         BER_BVZERO( &msc->msc_bound_ndn );
 458                 }
 459                 if ( !BER_BVISEMPTY( &op->o_ndn )
 460                         && SLAP_IS_AUTHZ_BACKEND( op )
 461                         && isauthz )
 462                 {
 463                         dc.target = mt;
 464                         dc.conn = op->o_conn;
 465                         dc.rs = rs;
 466                         dc.ctx = "bindDN";
 467
 468                         /*
 469                          * Rewrite the bind dn if needed
 470                          */
 471                         if ( asyncmeta_dn_massage( &dc, &op->o_conn->c_dn,
 472                                                 &msc->msc_bound_ndn ) )
 473                         {
 474
 475 #ifdef DEBUG_205
 476                                 Debug( LDAP_DEBUG_ANY,
 477                                         "### %s asyncmeta_init_one_conn(rewrite) "
 478                                         "ldap_unbind_ext[%d] ld=%p\n",
 479                                         op->o_log_prefix, candidate,
 480                                         (void *)msc->msc_ld );
 481 #endif /* DEBUG_205 */
 482                                 goto error_return;
 483                         }
 484
 485                         /* copy the DN if needed */
 486                         if ( msc->msc_bound_ndn.bv_val == op->o_conn->c_dn.bv_val ) {
 487                                 ber_dupbv( &msc->msc_bound_ndn, &op->o_conn->c_dn );
 488                         }
 489
 490                         assert( !BER_BVISNULL( &msc->msc_bound_ndn ) );
 491
 492                 } else {
 493                         ber_dupbv( &msc->msc_bound_ndn, (struct berval *)&slap_empty_bv );
 494                 }
 495         }
 496         assert( !BER_BVISNULL( &msc->msc_bound_ndn ) );
 497
 498 error_return:;
 499
 500         if (msc != NULL) {
 501                 META_BACK_CONN_CREATING_CLEAR( msc );
 502         }
 503         if ( rs->sr_err == LDAP_SUCCESS && msc != NULL) {
 504                 /*
 505                  * Sets a cookie for the rewrite session
 506                  */
 507                 ( void )rewrite_session_init( mt->mt_rwmap.rwm_rw, op->o_conn );
 508                 META_BACK_CONN_INITED_SET( msc );
 509         }
 510
 511         if ( rs->sr_err != LDAP_SUCCESS ) {
 512                 rs->sr_err = slap_map_api2result( rs );
 513                 if ( sendok & LDAP_BACK_SENDERR ) {
 514                         send_ldap_result( op, rs );
 515                 }
 516         }
 517         return rs->sr_err;
 518 }
 519
 520 /*
 521  * asyncmeta_retry
 522  *
 523  * Retries one connection
 524  */
 525 int
 526 asyncmeta_retry(
 527         Operation               *op,
 528         SlapReply               *rs,
 529         a_metaconn_t            **mcp,
 530         int                     candidate,
 531         ldap_back_send_t        sendok )
 532 {
 533         a_metaconn_t            *mc = *mcp;
 534         a_metainfo_t            *mi = mc->mc_info;
 535         a_metatarget_t          *mt = mi->mi_targets[ candidate ];
 536         a_metasingleconn_t      *msc = &mc->mc_conns[ candidate ];
 537         int                     rc = LDAP_UNAVAILABLE,
 538                                 binding,
 539                                 quarantine = 1;
 540
 541         ldap_pvt_thread_mutex_lock( &mc->mc_om_mutex );
 542         struct berval save_cred;
 543
 544         if ( LogTest( LDAP_DEBUG_ANY ) ) {
 545                 char    buf[ SLAP_TEXT_BUFLEN ];
 546
 547                 /* this lock is required; however,
 548                  * it's invoked only when logging is on */
 549                         ldap_pvt_thread_mutex_lock( &mt->mt_uri_mutex );
 550                         snprintf( buf, sizeof( buf ),
 551                                   "retrying URI=\"%s\" DN=\"%s\"",
 552                                   mt->mt_uri,
 553                                   BER_BVISNULL( &msc->msc_bound_ndn ) ?
 554                                   "" : msc->msc_bound_ndn.bv_val );
 555                         ldap_pvt_thread_mutex_unlock( &mt->mt_uri_mutex );
 556
 557                         Debug( LDAP_DEBUG_ANY,
 558                                "%s asyncmeta_retry[%d]: %s.\n",
 559                                op->o_log_prefix, candidate, buf );
 560         }
 561
 562         ( void )rewrite_session_delete( mt->mt_rwmap.rwm_rw, op->o_conn );
 563         /* mc here must be the regular mc, reset and ready for init */
 564         rc = asyncmeta_init_one_conn( op, rs, mc, candidate,
 565                                       LDAP_BACK_CONN_ISPRIV( mc ), sendok, 0 );
 566
 567         if ( rc != LDAP_SUCCESS ) {
 568                 if ( sendok & LDAP_BACK_SENDERR ) {
 569                         /* init_one_conn has set the result */
 570                         send_ldap_result( op, rs );
 571                 }
 572         }
 573
 574         if ( rc == LDAP_SUCCESS ) {
 575                 quarantine = 0;
 576                 LDAP_BACK_CONN_BINDING_SET( msc ); binding = 1;
 577                 /* todo this must be dobind_init */
 578                 rc = asyncmeta_back_single_dobind( op, rs, mcp, candidate,
 579                                                    sendok, mt->mt_nretries, 0 );
 580
 581                 Debug( LDAP_DEBUG_ANY,
 582                        "%s asyncmeta_retry[%d]: "
 583                        "asyncmeta_single_dobind=%d\n",
 584                        op->o_log_prefix, candidate, rc );
 585                 if ( rc == LDAP_SUCCESS ) {
 586                         if ( !BER_BVISNULL( &msc->msc_bound_ndn ) &&
 587                              !BER_BVISEMPTY( &msc->msc_bound_ndn ) )
 588                         {
 589                                 LDAP_BACK_CONN_ISBOUND_SET( msc );
 590
 591                         } else {
 592                                 LDAP_BACK_CONN_ISANON_SET( msc );
 593                         }
 594
 595                         /* when bound, dispose of the "binding" flag */
 596                         if ( binding ) {
 597                                 LDAP_BACK_CONN_BINDING_CLEAR( msc );
 598                         }
 599                 }
 600         }
 601
 602
 603         if ( rc != LDAP_SUCCESS ) {
 604                 if (mc->mc_active < 1) {
 605                         asyncmeta_clear_one_msc(NULL, mc, candidate);
 606                 }
 607                 if ( sendok & LDAP_BACK_SENDERR ) {
 608                         rs->sr_err = rc;
 609                         rs->sr_text = "Unable to retry";
 610                         send_ldap_result( op, rs );
 611                 }
 612         }
 613
 614         ldap_pvt_thread_mutex_unlock( &mc->mc_om_mutex );
 615         if ( quarantine && META_BACK_TGT_QUARANTINE( mt ) ) {
 616                 asyncmeta_quarantine( op, mi, rs, candidate );
 617         }
 618
 619         return rc == LDAP_SUCCESS ? 1 : 0;
 620 }
 621
 622 /*
 623  * callback for unique candidate selection
 624  */
 625 static int
 626 asyncmeta_conn_cb( Operation *op, SlapReply *rs )
 627 {
 628         assert( op->o_tag == LDAP_REQ_SEARCH );
 629
 630         switch ( rs->sr_type ) {
 631         case REP_SEARCH:
 632                 ((long *)op->o_callback->sc_private)[0] = (long)op->o_private;
 633                 break;
 634
 635         case REP_SEARCHREF:
 636         case REP_RESULT:
 637                 break;
 638
 639         default:
 640                 return rs->sr_err;
 641         }
 642
 643         return 0;
 644 }
 645
 646
 647 static int
 648 asyncmeta_get_candidate(
 649         Operation       *op,
 650         SlapReply       *rs,
 651         struct berval   *ndn )
 652 {
 653         a_metainfo_t    *mi = ( a_metainfo_t * )op->o_bd->be_private;
 654         long            candidate;
 655
 656         /*
 657          * tries to get a unique candidate
 658          * (takes care of default target)
 659          */
 660         candidate = asyncmeta_select_unique_candidate( mi, ndn );
 661
 662         /*
 663          * if any is found, inits the connection
 664          */
 665         if ( candidate == META_TARGET_NONE ) {
 666                 rs->sr_err = LDAP_NO_SUCH_OBJECT;
 667                 rs->sr_text = "No suitable candidate target found";
 668
 669         } else if ( candidate == META_TARGET_MULTIPLE ) {
 670                 Operation       op2 = *op;
 671                 SlapReply       rs2 = { REP_RESULT };
 672                 slap_callback   cb2 = { 0 };
 673                 int             rc;
 674
 675                 /* try to get a unique match for the request ndn
 676                  * among the multiple candidates available */
 677                 op2.o_tag = LDAP_REQ_SEARCH;
 678                 op2.o_req_dn = *ndn;
 679                 op2.o_req_ndn = *ndn;
 680                 op2.ors_scope = LDAP_SCOPE_BASE;
 681                 op2.ors_deref = LDAP_DEREF_NEVER;
 682                 op2.ors_attrs = slap_anlist_no_attrs;
 683                 op2.ors_attrsonly = 0;
 684                 op2.ors_limit = NULL;
 685                 op2.ors_slimit = 1;
 686                 op2.ors_tlimit = SLAP_NO_LIMIT;
 687
 688                 op2.ors_filter = (Filter *)slap_filter_objectClass_pres;
 689                 op2.ors_filterstr = *slap_filterstr_objectClass_pres;
 690
 691                 op2.o_callback = &cb2;
 692                 cb2.sc_response = asyncmeta_conn_cb;
 693                 cb2.sc_private = (void *)&candidate;
 694
 695                 rc = op->o_bd->be_search( &op2, &rs2 );
 696
 697                 switch ( rs2.sr_err ) {
 698                 case LDAP_SUCCESS:
 699                 default:
 700                         rs->sr_err = rs2.sr_err;
 701                         break;
 702
 703                 case LDAP_SIZELIMIT_EXCEEDED:
 704                         /* if multiple candidates can serve the operation,
 705                          * and a default target is defined, and it is
 706                          * a candidate, try using it (FIXME: YMMV) */
 707                         if ( mi->mi_defaulttarget != META_DEFAULT_TARGET_NONE
 708                                 && asyncmeta_is_candidate( mi->mi_targets[ mi->mi_defaulttarget ],
 709                                                 ndn, op->o_tag == LDAP_REQ_SEARCH ? op->ors_scope : LDAP_SCOPE_BASE ) )
 710                         {
 711                                 candidate = mi->mi_defaulttarget;
 712                                 rs->sr_err = LDAP_SUCCESS;
 713                                 rs->sr_text = NULL;
 714
 715                         } else {
 716                                 rs->sr_err = LDAP_UNWILLING_TO_PERFORM;
 717                                 rs->sr_text = "Unable to select unique candidate target";
 718                         }
 719                         break;
 720                 }
 721
 722         } else {
 723                 rs->sr_err = LDAP_SUCCESS;
 724         }
 725
 726         return candidate;
 727 }
 728
 729 static void     *asyncmeta_candidates_dummy;
 730
 731 static void
 732 asyncmeta_candidates_keyfree(
 733         void            *key,
 734         void            *data )
 735 {
 736         a_metacandidates_t      *mc = (a_metacandidates_t *)data;
 737
 738         ber_memfree_x( mc->mc_candidates, NULL );
 739         ber_memfree_x( data, NULL );
 740 }
 741
 742
 743 /*
 744  * asyncmeta_getconn
 745  *
 746  * Prepares the connection structure
 747  *
 748  * RATIONALE:
 749  *
 750  * - determine what DN is being requested:
 751  *
 752  *      op      requires candidate      checks
 753  *
 754  *      add     unique                  parent of o_req_ndn
 755  *      bind    unique^*[/all]          o_req_ndn [no check]
 756  *      compare unique^+                o_req_ndn
 757  *      delete  unique                  o_req_ndn
 758  *      modify  unique                  o_req_ndn
 759  *      search  any                     o_req_ndn
 760  *      modrdn  unique[, unique]        o_req_ndn[, orr_nnewSup]
 761  *
 762  * - for ops that require the candidate to be unique, in case of multiple
 763  *   occurrences an internal search with sizeLimit=1 is performed
 764  *   if a unique candidate can actually be determined.  If none is found,
 765  *   the operation aborts; if multiple are found, the default target
 766  *   is used if defined and candidate; otherwise the operation aborts.
 767  *
 768  * *^note: actually, the bind operation is handled much like a search;
 769  *   i.e. the bind is broadcast to all candidate targets.
 770  *
 771  * +^note: actually, the compare operation is handled much like a search;
 772  *   i.e. the compare is broadcast to all candidate targets, while checking
 773  *   that exactly none (noSuchObject) or one (TRUE/FALSE/UNDEFINED) is
 774  *   returned.
 775  */
 776 a_metaconn_t *
 777 asyncmeta_getconn(
 778         Operation               *op,
 779         SlapReply               *rs,
 780         SlapReply               *candidates,
 781         int                     *candidate,
 782         ldap_back_send_t        sendok,
 783         int                     alloc_new)
 784 {
 785         a_metainfo_t    *mi = ( a_metainfo_t * )op->o_bd->be_private;
 786         a_metaconn_t    *mc = NULL,
 787                         mc_curr = {{ 0 }};
 788         int             cached = META_TARGET_NONE,
 789                         i = META_TARGET_NONE,
 790                         err = LDAP_SUCCESS,
 791                         new_conn = 0,
 792                         ncandidates = 0;
 793
 794
 795         meta_op_type    op_type = META_OP_REQUIRE_SINGLE;
 796         enum            {
 797                 META_DNTYPE_ENTRY,
 798                 META_DNTYPE_PARENT,
 799                 META_DNTYPE_NEWPARENT
 800         }               dn_type = META_DNTYPE_ENTRY;
 801         struct berval   ndn = op->o_req_ndn,
 802                         pndn;
 803
 804         if (alloc_new > 0) {
 805                 mc = asyncmeta_conn_alloc(mi);
 806                 new_conn = 0;
 807         } else {
 808                 mc = asyncmeta_get_next_mc(mi);
 809         }
 810
 811         ldap_pvt_thread_mutex_lock(&mc->mc_om_mutex);
 812         /* Internal searches are privileged and shared. So is root. */
 813         if ( ( !BER_BVISEMPTY( &op->o_ndn ) && META_BACK_PROXYAUTHZ_ALWAYS( mi ) )
 814                 || ( BER_BVISEMPTY( &op->o_ndn ) && META_BACK_PROXYAUTHZ_ANON( mi ) )
 815                 || op->o_do_not_cache || be_isroot( op ) )
 816         {
 817                 LDAP_BACK_CONN_ISPRIV_SET( &mc_curr );
 818                 LDAP_BACK_PCONN_ROOTDN_SET( &mc_curr, op );
 819
 820         } else if ( BER_BVISEMPTY( &op->o_ndn ) && META_BACK_PROXYAUTHZ_NOANON( mi ) )
 821         {
 822                 LDAP_BACK_CONN_ISANON_SET( &mc_curr );
 823                 LDAP_BACK_PCONN_ANON_SET( &mc_curr, op );
 824
 825         } else {
 826                 /* Explicit binds must not be shared */
 827                 if ( !BER_BVISEMPTY( &op->o_ndn )
 828                         || op->o_tag == LDAP_REQ_BIND
 829                         || SLAP_IS_AUTHZ_BACKEND( op ) )
 830                 {
 831                         //mc_curr.mc_conn = op->o_conn;
 832
 833                 } else {
 834                         LDAP_BACK_CONN_ISANON_SET( &mc_curr );
 835                         LDAP_BACK_PCONN_ANON_SET( &mc_curr, op );
 836                 }
 837         }
 838
 839         switch ( op->o_tag ) {
 840         case LDAP_REQ_ADD:
 841                 /* if we go to selection, the entry must not exist,
 842                  * and we must be able to resolve the parent */
 843                 dn_type = META_DNTYPE_PARENT;
 844                 dnParent( &ndn, &pndn );
 845                 break;
 846
 847         case LDAP_REQ_MODRDN:
 848                 /* if nnewSuperior is not NULL, it must resolve
 849                  * to the same candidate as the req_ndn */
 850                 if ( op->orr_nnewSup ) {
 851                         dn_type = META_DNTYPE_NEWPARENT;
 852                 }
 853                 break;
 854
 855         case LDAP_REQ_BIND:
 856                 /* if bound as rootdn, the backend must bind to all targets
 857                  * with the administrative identity
 858                  * (unless pseoudoroot-bind-defer is TRUE) */
 859                 if ( op->orb_method == LDAP_AUTH_SIMPLE && be_isroot_pw( op ) ) {
 860                         op_type = META_OP_REQUIRE_ALL;
 861                 }
 862                 break;
 863
 864         case LDAP_REQ_COMPARE:
 865         case LDAP_REQ_DELETE:
 866         case LDAP_REQ_MODIFY:
 867                 /* just a unique candidate */
 868                 break;
 869
 870         case LDAP_REQ_SEARCH:
 871                 /* allow multiple candidates for the searchBase */
 872                 op_type = META_OP_ALLOW_MULTIPLE;
 873                 break;
 874
 875         default:
 876                 /* right now, just break (exop?) */
 877                 break;
 878         }
 879
 880         /*
 881          * require all connections ...
 882          */
 883         if ( op_type == META_OP_REQUIRE_ALL ) {
 884                 if ( LDAP_BACK_CONN_ISPRIV( &mc_curr ) ) {
 885                         LDAP_BACK_CONN_ISPRIV_SET( mc );
 886
 887                 } else if ( LDAP_BACK_CONN_ISANON( &mc_curr ) ) {
 888                         LDAP_BACK_CONN_ISANON_SET( mc );
 889                 }
 890
 891                 for ( i = 0; i < mi->mi_ntargets; i++ ) {
 892                         /*
 893                          * The target is activated; if needed, it is
 894                          * also init'd
 895                          */
 896                         candidates[ i ].sr_err = asyncmeta_init_one_conn( op,
 897                                 rs, mc, i, LDAP_BACK_CONN_ISPRIV( &mc_curr ),
 898                                 LDAP_BACK_DONTSEND, !new_conn );
 899                         if ( candidates[ i ].sr_err == LDAP_SUCCESS ) {
 900                                 if ( new_conn && ( sendok & LDAP_BACK_BINDING ) ) {
 901                                         LDAP_BACK_CONN_BINDING_SET( &mc->mc_conns[ i ] );
 902                                 }
 903                                 META_CANDIDATE_SET( &candidates[ i ] );
 904                                 ncandidates++;
 905
 906                         } else {
 907
 908                                 /*
 909                                  * FIXME: in case one target cannot
 910                                  * be init'd, should the other ones
 911                                  * be tried?
 912                                  */
 913                                 META_CANDIDATE_RESET( &candidates[ i ] );
 914                                 err = candidates[ i ].sr_err;
 915                                 continue;
 916                         }
 917                 }
 918
 919                 if ( ncandidates == 0 ) {
 920                         rs->sr_err = LDAP_NO_SUCH_OBJECT;
 921                         rs->sr_text = "Unable to select valid candidates";
 922
 923                         if ( sendok & LDAP_BACK_SENDERR ) {
 924                                 if ( rs->sr_err == LDAP_NO_SUCH_OBJECT ) {
 925                                         rs->sr_matched = op->o_bd->be_suffix[ 0 ].bv_val;
 926                                 }
 927                                 send_ldap_result( op, rs );
 928                                 rs->sr_matched = NULL;
 929                         }
 930                         ldap_pvt_thread_mutex_unlock(&mc->mc_om_mutex);
 931                         if ( alloc_new > 0) {
 932                                 asyncmeta_back_conn_free( mc );
 933                         }
 934                         return NULL;
 935                 }
 936
 937                 goto done;
 938         }
 939
 940         /*
 941          * looks in cache, if any
 942          */
 943         if ( mi->mi_cache.ttl != META_DNCACHE_DISABLED ) {
 944                 cached = i = asyncmeta_dncache_get_target( &mi->mi_cache, &op->o_req_ndn );
 945         }
 946
 947         if ( op_type == META_OP_REQUIRE_SINGLE ) {
 948                 a_metatarget_t          *mt = NULL;
 949                 a_metasingleconn_t      *msc = NULL;
 950
 951                 int                     j;
 952
 953                 for ( j = 0; j < mi->mi_ntargets; j++ ) {
 954                         META_CANDIDATE_RESET( &candidates[ j ] );
 955                 }
 956
 957                 /*
 958                  * tries to get a unique candidate
 959                  * (takes care of default target)
 960                  */
 961                 if ( i == META_TARGET_NONE ) {
 962                         i = asyncmeta_get_candidate( op, rs, &ndn );
 963
 964                         if ( rs->sr_err == LDAP_NO_SUCH_OBJECT && dn_type == META_DNTYPE_PARENT ) {
 965                                 i = asyncmeta_get_candidate( op, rs, &pndn );
 966                         }
 967
 968                         if ( i < 0 || rs->sr_err != LDAP_SUCCESS ) {
 969                                 if ( sendok & LDAP_BACK_SENDERR ) {
 970                                         if ( rs->sr_err == LDAP_NO_SUCH_OBJECT ) {
 971                                                 rs->sr_matched = op->o_bd->be_suffix[ 0 ].bv_val;
 972                                         }
 973                                         send_ldap_result( op, rs );
 974                                         rs->sr_matched = NULL;
 975                                 }
 976                                 ldap_pvt_thread_mutex_unlock(&mc->mc_om_mutex);
 977                                 if ( mc != NULL && alloc_new ) {
 978                                         asyncmeta_back_conn_free( mc );
 979                                 }
 980                                 return NULL;
 981                         }
 982                 }
 983
 984                 if ( dn_type == META_DNTYPE_NEWPARENT && asyncmeta_get_candidate( op, rs, op->orr_nnewSup ) != i )
 985                 {
 986                         rs->sr_err = LDAP_UNWILLING_TO_PERFORM;
 987                         rs->sr_text = "Cross-target rename not supported";
 988                         if ( sendok & LDAP_BACK_SENDERR ) {
 989                                 send_ldap_result( op, rs );
 990                         }
 991                         ldap_pvt_thread_mutex_unlock(&mc->mc_om_mutex);
 992                         if ( mc != NULL && alloc_new > 0 ) {
 993                                 asyncmeta_back_conn_free( mc );
 994                         }
 995                         return NULL;
 996                 }
 997
 998                 Debug( LDAP_DEBUG_TRACE,
 999                        "==>asyncmeta__getconn: got target=%d for ndn=\"%s\" from cache\n",
1000                        i, op->o_req_ndn.bv_val, 0 );
1001                         if ( LDAP_BACK_CONN_ISPRIV( &mc_curr ) ) {
1002                                 LDAP_BACK_CONN_ISPRIV_SET( mc );
1003
1004                         } else if ( LDAP_BACK_CONN_ISANON( &mc_curr ) ) {
1005                                 LDAP_BACK_CONN_ISANON_SET( mc );
1006                         }
1007
1008                 /*
1009                  * Clear all other candidates
1010                  */
1011                         ( void )asyncmeta_clear_unused_candidates( op, i , mc, candidates);
1012
1013                 mt = mi->mi_targets[ i ];
1014                 msc = &mc->mc_conns[ i ];
1015
1016                 /*
1017                  * The target is activated; if needed, it is
1018                  * also init'd. In case of error, asyncmeta_init_one_conn
1019                  * sends the appropriate result.
1020                  */
1021                 err = asyncmeta_init_one_conn( op, rs, mc, i,
1022                         LDAP_BACK_CONN_ISPRIV( &mc_curr ), sendok, !new_conn );
1023                 if ( err != LDAP_SUCCESS ) {
1024                         /*
1025                          * FIXME: in case one target cannot
1026                          * be init'd, should the other ones
1027                          * be tried?
1028                          */
1029                         META_CANDIDATE_RESET( &candidates[ i ] );
1030                         ldap_pvt_thread_mutex_unlock(&mc->mc_om_mutex);
1031                         if ( mc != NULL && alloc_new > 0 ) {
1032                                 asyncmeta_back_conn_free( mc );
1033                         }
1034                         return NULL;
1035                 }
1036
1037                 candidates[ i ].sr_err = LDAP_SUCCESS;
1038                 META_CANDIDATE_SET( &candidates[ i ] );
1039                 ncandidates++;
1040
1041                 if ( candidate ) {
1042                         *candidate = i;
1043                 }
1044
1045         /*
1046          * if no unique candidate ...
1047          */
1048         } else {
1049         if ( LDAP_BACK_CONN_ISPRIV( &mc_curr ) ) {
1050                 LDAP_BACK_CONN_ISPRIV_SET( mc );
1051
1052         } else if ( LDAP_BACK_CONN_ISANON( &mc_curr ) ) {
1053                 LDAP_BACK_CONN_ISANON_SET( mc );
1054         }
1055
1056                 for ( i = 0; i < mi->mi_ntargets; i++ ) {
1057                         a_metatarget_t          *mt = mi->mi_targets[ i ];
1058
1059                         META_CANDIDATE_RESET( &candidates[ i ] );
1060
1061                         if ( i == cached
1062                                 || asyncmeta_is_candidate( mt, &op->o_req_ndn,
1063                                         op->o_tag == LDAP_REQ_SEARCH ? op->ors_scope : LDAP_SCOPE_SUBTREE ) )
1064                         {
1065
1066                                 /*
1067                                  * The target is activated; if needed, it is
1068                                  * also init'd
1069                                  */
1070                                 int lerr = asyncmeta_init_one_conn( op, rs, mc, i,
1071                                         LDAP_BACK_CONN_ISPRIV( &mc_curr ),
1072                                         LDAP_BACK_DONTSEND, !new_conn );
1073                                 candidates[ i ].sr_err = lerr;
1074                                 if ( lerr == LDAP_SUCCESS ) {
1075                                         META_CANDIDATE_SET( &candidates[ i ] );
1076                                         ncandidates++;
1077
1078                                         Debug( LDAP_DEBUG_TRACE, "%s: asyncmeta_getconn[%d]\n",
1079                                                 op->o_log_prefix, i, 0 );
1080
1081                                 } else if ( lerr == LDAP_UNAVAILABLE && !META_BACK_ONERR_STOP( mi ) ) {
1082                                         META_CANDIDATE_SET( &candidates[ i ] );
1083
1084                                         Debug( LDAP_DEBUG_TRACE, "%s: asyncmeta_getconn[%d] %s\n",
1085                                                 op->o_log_prefix, i,
1086                                                 mt->mt_isquarantined != LDAP_BACK_FQ_NO ? "quarantined" : "unavailable" );
1087
1088                                 } else {
1089
1090                                         /*
1091                                          * FIXME: in case one target cannot
1092                                          * be init'd, should the other ones
1093                                          * be tried?
1094                                          */
1095                                         /* leave the target candidate, but record the error for later use */
1096                                         err = lerr;
1097
1098                                         if ( lerr == LDAP_UNAVAILABLE && mt->mt_isquarantined != LDAP_BACK_FQ_NO ) {
1099                                                 Debug( LDAP_DEBUG_TRACE, "%s: asyncmeta_getconn[%d] quarantined err=%d\n",
1100                                                         op->o_log_prefix, i, lerr );
1101
1102                                         } else {
1103                                                 Debug( LDAP_DEBUG_ANY, "%s: asyncmeta_getconn[%d] failed err=%d\n",
1104                                                         op->o_log_prefix, i, lerr );
1105                                         }
1106
1107                                         if ( META_BACK_ONERR_STOP( mi ) ) {
1108                                                 if ( sendok & LDAP_BACK_SENDERR ) {
1109                                                         send_ldap_result( op, rs );
1110                                                 }
1111                                                 ldap_pvt_thread_mutex_unlock(&mc->mc_om_mutex);
1112                                                 if ( alloc_new > 0 ) {
1113                                                         asyncmeta_back_conn_free( mc );
1114
1115                                                 }
1116                                                 return NULL;
1117                                         }
1118
1119                                         continue;
1120                                 }
1121
1122                         }
1123                 }
1124
1125                 if ( ncandidates == 0 ) {
1126                         if ( rs->sr_err == LDAP_SUCCESS ) {
1127                                 rs->sr_err = LDAP_NO_SUCH_OBJECT;
1128                                 rs->sr_text = "Unable to select valid candidates";
1129                         }
1130
1131                         if ( sendok & LDAP_BACK_SENDERR ) {
1132                                 if ( rs->sr_err == LDAP_NO_SUCH_OBJECT ) {
1133                                         rs->sr_matched = op->o_bd->be_suffix[ 0 ].bv_val;
1134                                 }
1135                                 send_ldap_result( op, rs );
1136                                 rs->sr_matched = NULL;
1137                         }
1138                         if ( alloc_new > 0 ) {
1139                                 asyncmeta_back_conn_free( mc );
1140
1141                         }
1142                         ldap_pvt_thread_mutex_unlock(&mc->mc_om_mutex);
1143                         return NULL;
1144                 }
1145         }
1146
1147 done:;
1148         /* clear out meta_back_init_one_conn non-fatal errors */
1149         rs->sr_err = LDAP_SUCCESS;
1150         rs->sr_text = NULL;
1151
1152         if ( new_conn ) {
1153                 if ( !LDAP_BACK_PCONN_ISPRIV( mc ) ) {
1154                         /*
1155                          * Err could be -1 in case a duplicate metaconn is inserted
1156                          */
1157                         switch ( err ) {
1158                         case 0:
1159                                 break;
1160                         default:
1161                                 LDAP_BACK_CONN_CACHED_CLEAR( mc );
1162                                 if ( LogTest( LDAP_DEBUG_ANY ) ) {
1163                                         char buf[STRLENOF("4294967295U") + 1] = { 0 };
1164                                         mi->mi_ldap_extra->connid2str( &mc->mc_base, buf, sizeof(buf) );
1165
1166                                         Debug( LDAP_DEBUG_ANY,
1167                                                 "%s asyncmeta_getconn: candidates=%d conn=%s insert failed\n",
1168                                                 op->o_log_prefix, ncandidates, buf );
1169                                 }
1170
1171                                 asyncmeta_back_conn_free( mc );
1172
1173                                 rs->sr_err = LDAP_OTHER;
1174                                 rs->sr_text = "Proxy bind collision";
1175                                 if ( sendok & LDAP_BACK_SENDERR ) {
1176                                         send_ldap_result( op, rs );
1177                                 }
1178                                 return NULL;
1179                         }
1180                 }
1181
1182                 if ( LogTest( LDAP_DEBUG_TRACE ) ) {
1183                         char buf[STRLENOF("4294967295U") + 1] = { 0 };
1184                         mi->mi_ldap_extra->connid2str( &mc->mc_base, buf, sizeof(buf) );
1185
1186                         Debug( LDAP_DEBUG_TRACE,
1187                                 "%s asyncmeta_getconn: candidates=%d conn=%s inserted\n",
1188                                 op->o_log_prefix, ncandidates, buf );
1189                 }
1190
1191         } else {
1192                 if ( LogTest( LDAP_DEBUG_TRACE ) ) {
1193                         char buf[STRLENOF("4294967295U") + 1] = { 0 };
1194                         mi->mi_ldap_extra->connid2str( &mc->mc_base, buf, sizeof(buf) );
1195
1196                         Debug( LDAP_DEBUG_TRACE,
1197                                 "%s asyncmeta_getconn: candidates=%d conn=%s fetched\n",
1198                                 op->o_log_prefix, ncandidates, buf );
1199                 }
1200         }
1201         ldap_pvt_thread_mutex_unlock(&mc->mc_om_mutex);
1202         return mc;
1203 }
1204
1205 void
1206 asyncmeta_quarantine(
1207         Operation       *op,
1208         a_metainfo_t    *mi,
1209         SlapReply       *rs,
1210         int             candidate )
1211 {
1212         a_metatarget_t          *mt = mi->mi_targets[ candidate ];
1213
1214         slap_retry_info_t       *ri = &mt->mt_quarantine;
1215
1216         ldap_pvt_thread_mutex_lock( &mt->mt_quarantine_mutex );
1217
1218         if ( rs->sr_err == LDAP_UNAVAILABLE ) {
1219                 time_t  new_last = slap_get_time();
1220
1221                 switch ( mt->mt_isquarantined ) {
1222                 case LDAP_BACK_FQ_NO:
1223                         if ( ri->ri_last == new_last ) {
1224                                 goto done;
1225                         }
1226
1227                         Debug( LDAP_DEBUG_ANY,
1228                                 "%s asyncmeta_quarantine[%d]: enter.\n",
1229                                 op->o_log_prefix, candidate, 0 );
1230
1231                         ri->ri_idx = 0;
1232                         ri->ri_count = 0;
1233                         break;
1234
1235                 case LDAP_BACK_FQ_RETRYING:
1236                         if ( LogTest( LDAP_DEBUG_ANY ) ) {
1237                                 char    buf[ SLAP_TEXT_BUFLEN ];
1238
1239                                 snprintf( buf, sizeof( buf ),
1240                                         "asyncmeta_quarantine[%d]: block #%d try #%d failed",
1241                                         candidate, ri->ri_idx, ri->ri_count );
1242                                 Debug( LDAP_DEBUG_ANY, "%s %s.\n",
1243                                         op->o_log_prefix, buf, 0 );
1244                         }
1245
1246                         ++ri->ri_count;
1247                         if ( ri->ri_num[ ri->ri_idx ] != SLAP_RETRYNUM_FOREVER
1248                                 && ri->ri_count == ri->ri_num[ ri->ri_idx ] )
1249                         {
1250                                 ri->ri_count = 0;
1251                                 ++ri->ri_idx;
1252                         }
1253                         break;
1254
1255                 default:
1256                         break;
1257                 }
1258
1259                 mt->mt_isquarantined = LDAP_BACK_FQ_YES;
1260                 ri->ri_last = new_last;
1261
1262         } else if ( mt->mt_isquarantined == LDAP_BACK_FQ_RETRYING ) {
1263                 Debug( LDAP_DEBUG_ANY,
1264                         "%s asyncmeta_quarantine[%d]: exit.\n",
1265                         op->o_log_prefix, candidate, 0 );
1266
1267                 if ( mi->mi_quarantine_f ) {
1268                         (void)mi->mi_quarantine_f( mi, candidate,
1269                                 mi->mi_quarantine_p );
1270                 }
1271
1272                 ri->ri_count = 0;
1273                 ri->ri_idx = 0;
1274                 mt->mt_isquarantined = LDAP_BACK_FQ_NO;
1275                 mt->mt_timeout_ops = 0;
1276         }
1277
1278 done:;
1279         ldap_pvt_thread_mutex_unlock( &mt->mt_quarantine_mutex );
1280 }
1281
1282 a_metaconn_t *
1283 asyncmeta_get_next_mc( a_metainfo_t *mi )
1284 {
1285         a_metaconn_t *mc = NULL;
1286         ldap_pvt_thread_mutex_lock( &mi->mi_mc_mutex );
1287         if (mi->mi_next_conn >= mi->mi_num_conns-1) {
1288                 mi->mi_next_conn = 0;
1289         } else {
1290                 mi->mi_next_conn++;
1291         }
1292
1293         mc = &mi->mi_conns[mi->mi_next_conn];
1294         ldap_pvt_thread_mutex_unlock( &mi->mi_mc_mutex );
1295         return mc;
1296 }
1297
1298 int asyncmeta_start_listeners(a_metaconn_t *mc, SlapReply *candidates, bm_context_t *bc)
1299 {
1300         int i;
1301         for (i = 0; i < mc->mc_info->mi_ntargets; i++) {
1302                 asyncmeta_start_one_listener(mc, candidates, bc, i);
1303         }
1304         return LDAP_SUCCESS;
1305 }
1306
1307 int asyncmeta_start_one_listener(a_metaconn_t *mc,
1308                                  SlapReply *candidates,
1309                                  bm_context_t *bc,
1310                                  int candidate)
1311 {
1312         a_metasingleconn_t *msc;
1313         ber_socket_t s;
1314         int i;
1315
1316         msc = &mc->mc_conns[candidate];
1317         if (msc->msc_ld == NULL || !META_IS_CANDIDATE( &candidates[ candidate ] )) {
1318                 return LDAP_SUCCESS;
1319         }
1320         bc->msgids[candidate] = candidates[candidate].sr_msgid;
1321         msc->msc_pending_ops++;
1322         if ( msc->conn == NULL) {
1323                 ldap_get_option( msc->msc_ld, LDAP_OPT_DESC, &s );
1324                 if (s < 0) {
1325                         /* Todo a meaningful log pls */
1326                         return LDAP_OTHER;
1327                 }
1328                 msc->conn = connection_client_setup( s, asyncmeta_op_handle_result, mc );
1329         }
1330         connection_client_enable( msc->conn );
1331         return LDAP_SUCCESS;
1332 }