X-Git-Url: https://git.sur5r.net/?a=blobdiff_plain;f=servers%2Fslapd%2Fsyncrepl.c;h=dff0087e4660f73cd798c7bd898bb843e41b828d;hb=f5100665e34b18817e545063b7ca49f79a15b6d6;hp=1e6e00c049b611343ed9b633b34dfc4de1d9a420;hpb=0b03c51e79e714b8f8c8d6811f942f52c52b2383;p=openldap diff --git a/servers/slapd/syncrepl.c b/servers/slapd/syncrepl.c index 1e6e00c049..dff0087e46 100644 --- a/servers/slapd/syncrepl.c +++ b/servers/slapd/syncrepl.c @@ -2,7 +2,7 @@ /* $OpenLDAP$ */ /* This work is part of OpenLDAP Software . * - * Copyright 2003-2013 The OpenLDAP Foundation. + * Copyright 2003-2015 The OpenLDAP Foundation. * Portions Copyright 2003 by IBM Corporation. * Portions Copyright 2003-2008 by Howard Chu, Symas Corporation. * All rights reserved. @@ -36,6 +36,8 @@ #define SUFFIXM_CTX "" #endif +#define UUIDLEN 16 + struct nonpresent_entry { struct berval *npe_name; struct berval *npe_nname; @@ -108,8 +110,13 @@ typedef struct syncinfo_s { int si_refreshDelete; int si_refreshPresent; int si_refreshDone; + int si_refreshCount; + time_t si_refreshBeg; + time_t si_refreshEnd; + OpExtra *si_refreshTxn; int si_syncdata; int si_logstate; + int si_lazyCommit; int si_got; int si_strict_refresh; /* stop listening during fallback refresh */ int si_too_old; @@ -126,7 +133,10 @@ typedef struct syncinfo_s { } syncinfo_t; static int syncuuid_cmp( const void *, const void * ); -static int avl_presentlist_insert( syncinfo_t* si, struct berval *syncUUID ); +static int presentlist_insert( syncinfo_t* si, struct berval *syncUUID ); +static void presentlist_delete( Avlnode **av, struct berval *syncUUID ); +static char *presentlist_find( Avlnode *av, struct berval *syncUUID ); +static int presentlist_free( Avlnode *av ); static void syncrepl_del_nonpresent( Operation *, syncinfo_t *, BerVarray, struct sync_cookie *, int ); static int syncrepl_message_to_op( syncinfo_t *, Operation *, LDAPMessage * ); @@ -713,6 +723,7 @@ do_syncrep1( ldap_pvt_thread_mutex_unlock( &si->si_cookieState->cs_mutex ); } + ch_free( si->si_syncCookie.octet_str.bv_val ); slap_compose_sync_cookie( NULL, &si->si_syncCookie.octet_str, si->si_syncCookie.ctxcsn, si->si_syncCookie.rid, si->si_syncCookie.sid ); @@ -729,6 +740,11 @@ do_syncrep1( } si->si_refreshDone = 0; + si->si_refreshBeg = slap_get_time(); + si->si_refreshCount = 0; + si->si_refreshTxn = NULL; + Debug( LDAP_DEBUG_ANY, "do_syncrep1: %s starting refresh\n", + si->si_ridtxt, 0, 0 ); rc = ldap_sync_search( si, op->o_tmpmemctx ); @@ -898,10 +914,10 @@ do_syncrep2( } /* FIXME: what if syncUUID is NULL or empty? * (happens with back-sql...) */ - if ( BER_BVISEMPTY( &syncUUID[0] ) ) { + if ( syncUUID[0].bv_len != UUIDLEN ) { bdn.bv_val[bdn.bv_len] = '\0'; Debug( LDAP_DEBUG_ANY, "do_syncrep2: %s " - "got empty syncUUID with LDAP_SYNC_%s (%s)\n", + "got empty or invalid syncUUID with LDAP_SYNC_%s (%s)\n", si->si_ridtxt, syncrepl_state2str( syncstate ), bdn.bv_val ); ldap_controls_free( rctrls ); @@ -910,7 +926,7 @@ do_syncrep2( } punlock = -1; if ( ber_peek_tag( ber, &len ) == LDAP_TAG_SYNC_COOKIE ) { - ber_scanf( ber, /*"{"*/ "m}", &cookie ); + if ( ber_scanf( ber, /*"{"*/ "m}", &cookie ) != LBER_ERROR ) { Debug( LDAP_DEBUG_SYNC, "do_syncrep2: %s cookie=%s\n", si->si_ridtxt, @@ -926,6 +942,7 @@ do_syncrep2( if ( syncCookie.ctxcsn ) { int i, sid = slap_parse_csn_sid( syncCookie.ctxcsn ); check_syncprov( op, si ); + ldap_pvt_thread_mutex_lock( &si->si_cookieState->cs_mutex ); for ( i =0; isi_cookieState->cs_num; i++ ) { /* new SID */ if ( sid < si->si_cookieState->cs_sids[i] ) @@ -935,15 +952,18 @@ do_syncrep2( bdn.bv_val[bdn.bv_len] = '\0'; Debug( LDAP_DEBUG_SYNC, "do_syncrep2: %s CSN too old, ignoring %s (%s)\n", si->si_ridtxt, syncCookie.ctxcsn->bv_val, bdn.bv_val ); + si->si_too_old = 1; + ldap_pvt_thread_mutex_unlock( &si->si_cookieState->cs_mutex ); ldap_controls_free( rctrls ); rc = 0; - si->si_too_old = 1; goto done; } si->si_too_old = 0; break; } } + ldap_pvt_thread_mutex_unlock( &si->si_cookieState->cs_mutex ); + /* check pending CSNs too */ while ( ldap_pvt_thread_mutex_trylock( &si->si_cookieState->cs_pmutex )) { if ( slapd_shutdown ) { @@ -953,17 +973,18 @@ do_syncrep2( if ( !ldap_pvt_thread_pool_pausecheck( &connection_pool )) ldap_pvt_thread_yield(); } + for ( i =0; isi_cookieState->cs_pnum; i++ ) { if ( sid < si->si_cookieState->cs_psids[i] ) break; if ( si->si_cookieState->cs_psids[i] == sid ) { if ( ber_bvcmp( syncCookie.ctxcsn, &si->si_cookieState->cs_pvals[i] ) <= 0 ) { + ldap_pvt_thread_mutex_unlock( &si->si_cookieState->cs_pmutex ); bdn.bv_val[bdn.bv_len] = '\0'; Debug( LDAP_DEBUG_SYNC, "do_syncrep2: %s CSN pending, ignoring %s (%s)\n", si->si_ridtxt, syncCookie.ctxcsn->bv_val, bdn.bv_val ); ldap_controls_free( rctrls ); rc = 0; - ldap_pvt_thread_mutex_unlock( &si->si_cookieState->cs_pmutex ); goto done; } ber_bvreplace( &si->si_cookieState->cs_pvals[i], @@ -990,6 +1011,7 @@ do_syncrep2( } op->o_controls[slap_cids.sc_LDAPsync] = &syncCookie; } + } } rc = 0; if ( si->si_syncdata && si->si_logstate == SYNCLOG_LOGGING ) { @@ -1032,6 +1054,7 @@ do_syncrep2( /* on failure, revert pending CSN */ if ( rc != LDAP_SUCCESS ) { int i; + ldap_pvt_thread_mutex_lock( &si->si_cookieState->cs_mutex ); for ( i = 0; isi_cookieState->cs_num; i++ ) { if ( si->si_cookieState->cs_sids[i] == si->si_cookieState->cs_psids[punlock] ) { ber_bvreplace( &si->si_cookieState->cs_pvals[punlock], @@ -1041,6 +1064,7 @@ do_syncrep2( } if ( i == si->si_cookieState->cs_num ) si->si_cookieState->cs_pvals[punlock].bv_val[0] = '\0'; + ldap_pvt_thread_mutex_unlock( &si->si_cookieState->cs_mutex ); } ldap_pvt_thread_mutex_unlock( &si->si_cookieState->cs_pmutex ); } @@ -1162,8 +1186,8 @@ do_syncrep2( { syncrepl_del_nonpresent( op, si, NULL, &syncCookie, m ); - } else { - avl_free( si->si_presentlist, ch_free ); + } else if ( si->si_presentlist ) { + presentlist_free( si->si_presentlist ); si->si_presentlist = NULL; } } @@ -1171,6 +1195,13 @@ do_syncrep2( { rc = syncrepl_updateCookie( si, op, &syncCookie ); } + if ( si->si_refreshCount ) { + LDAP_SLIST_REMOVE( &op->o_extra, si->si_refreshTxn, OpExtra, oe_next ); + op->o_bd->bd_info->bi_op_txn( op, SLAP_TXN_COMMIT, &si->si_refreshTxn ); + si->si_refreshCount = 0; + si->si_refreshTxn = NULL; + } + si->si_refreshEnd = slap_get_time(); if ( err == LDAP_SUCCESS && si->si_logstate == SYNCLOG_FALLBACK ) { si->si_logstate = SYNCLOG_LOGGING; @@ -1252,6 +1283,17 @@ do_syncrep2( { si->si_refreshDone = 1; } + if ( si->si_refreshDone ) { + if ( si->si_refreshCount ) { + LDAP_SLIST_REMOVE( &op->o_extra, si->si_refreshTxn, OpExtra, oe_next ); + op->o_bd->bd_info->bi_op_txn( op, SLAP_TXN_COMMIT, &si->si_refreshTxn ); + si->si_refreshCount = 0; + si->si_refreshTxn = NULL; + } + si->si_refreshEnd = slap_get_time(); + Debug( LDAP_DEBUG_ANY, "do_syncrep1: %s finished refresh\n", + si->si_ridtxt, 0, 0 ); + } ber_scanf( ber, /*"{"*/ "}" ); if ( abs(si->si_type) == LDAP_SYNC_REFRESH_AND_PERSIST && si->si_refreshDone ) @@ -1290,20 +1332,23 @@ do_syncrep2( ber_scanf( ber, "b", &refreshDeletes ); } syncUUIDs = NULL; - ber_scanf( ber, "[W]", &syncUUIDs ); + rc = ber_scanf( ber, "[W]", &syncUUIDs ); ber_scanf( ber, /*"{"*/ "}" ); - if ( refreshDeletes ) { - syncrepl_del_nonpresent( op, si, syncUUIDs, - &syncCookie, m ); - ber_bvarray_free_x( syncUUIDs, op->o_tmpmemctx ); - } else { - int i; - for ( i = 0; !BER_BVISNULL( &syncUUIDs[i] ); i++ ) { - (void)avl_presentlist_insert( si, &syncUUIDs[i] ); - slap_sl_free( syncUUIDs[i].bv_val, op->o_tmpmemctx ); + if ( rc != LBER_ERROR ) { + if ( refreshDeletes ) { + syncrepl_del_nonpresent( op, si, syncUUIDs, + &syncCookie, m ); + ber_bvarray_free_x( syncUUIDs, op->o_tmpmemctx ); + } else { + int i; + for ( i = 0; !BER_BVISNULL( &syncUUIDs[i] ); i++ ) { + (void)presentlist_insert( si, &syncUUIDs[i] ); + slap_sl_free( syncUUIDs[i].bv_val, op->o_tmpmemctx ); + } + slap_sl_free( syncUUIDs, op->o_tmpmemctx ); } - slap_sl_free( syncUUIDs, op->o_tmpmemctx ); } + rc = 0; slap_sync_cookie_free( &syncCookie, 0 ); break; default: @@ -1340,6 +1385,10 @@ do_syncrep2( { rc = syncrepl_updateCookie( si, op, &syncCookie); } + if ( si->si_presentlist ) { + presentlist_free( si->si_presentlist ); + si->si_presentlist = NULL; + } } ldap_memfree( retoid ); @@ -1375,6 +1424,12 @@ do_syncrep2( if ( ldap_pvt_thread_pool_pausing( &connection_pool )) { slap_sync_cookie_free( &syncCookie, 0 ); slap_sync_cookie_free( &syncCookie_req, 0 ); + if ( si->si_refreshCount ) { + LDAP_SLIST_REMOVE( &op->o_extra, si->si_refreshTxn, OpExtra, oe_next ); + op->o_bd->bd_info->bi_op_txn( op, SLAP_TXN_COMMIT, &si->si_refreshTxn ); + si->si_refreshCount = 0; + si->si_refreshTxn = NULL; + } return SYNC_PAUSED; } } @@ -1516,7 +1571,7 @@ do_syncrepl( si->si_refreshPresent = 0; if ( si->si_presentlist ) { - avl_free( si->si_presentlist, ch_free ); + presentlist_free( si->si_presentlist ); si->si_presentlist = NULL; } @@ -1538,6 +1593,10 @@ reload: op->o_ndn = op->o_bd->be_rootndn; rc = do_syncrep2( op, si ); if ( rc == LDAP_SYNC_REFRESH_REQUIRED ) { + if ( BER_BVISNULL( &si->si_syncCookie.octet_str )) + slap_compose_sync_cookie( NULL, &si->si_syncCookie.octet_str, + si->si_syncCookie.ctxcsn, si->si_syncCookie.rid, + si->si_syncCookie.sid ); rc = ldap_sync_search( si, op->o_tmpmemctx ); goto reload; } @@ -2051,8 +2110,11 @@ syncrepl_op_modify( Operation *op, SlapReply *rs ) overlay_entry_release_ov( op, e, 0, on ); } /* equal? Should never happen */ - if ( match == 0 ) + if ( match == 0 ) { + /* tell accesslog this was a failure */ + rs->sr_err = LDAP_TYPE_OR_VALUE_EXISTS; return LDAP_SUCCESS; + } /* mod is older: resolve conflicts... * 1. Save/copy original modlist. Split Replace to Del/Add. @@ -2668,29 +2730,106 @@ typedef struct dninfo { AttributeDescription *newDesc; /* for renames */ } dninfo; +#define HASHUUID 1 + /* return 1 if inserted, 0 otherwise */ static int -avl_presentlist_insert( +presentlist_insert( syncinfo_t* si, struct berval *syncUUID ) { - struct berval *syncuuid_bv = ch_malloc( sizeof( struct berval ) + syncUUID->bv_len + 1 ); + char *val; + +#ifdef HASHUUID + Avlnode **av; + unsigned short s; + + if ( !si->si_presentlist ) + si->si_presentlist = ch_calloc(65536, sizeof( Avlnode * )); + + av = (Avlnode **)si->si_presentlist; + + val = ch_malloc(UUIDLEN-2); + memcpy(&s, syncUUID->bv_val, 2); + memcpy(val, syncUUID->bv_val+2, UUIDLEN-2); + + if ( avl_insert( &av[s], val, + syncuuid_cmp, avl_dup_error ) ) + { + ch_free( val ); + return 0; + } +#else + val = ch_malloc(UUIDLEN); - syncuuid_bv->bv_len = syncUUID->bv_len; - syncuuid_bv->bv_val = (char *)&syncuuid_bv[1]; - AC_MEMCPY( syncuuid_bv->bv_val, syncUUID->bv_val, syncUUID->bv_len ); - syncuuid_bv->bv_val[ syncuuid_bv->bv_len ] = '\0'; + AC_MEMCPY( val, syncUUID->bv_val, UUIDLEN ); - if ( avl_insert( &si->si_presentlist, (caddr_t) syncuuid_bv, + if ( avl_insert( &si->si_presentlist, val, syncuuid_cmp, avl_dup_error ) ) { - ch_free( syncuuid_bv ); + ch_free( val ); return 0; } +#endif return 1; } +static char * +presentlist_find( + Avlnode *av, + struct berval *val ) +{ +#ifdef HASHUUID + Avlnode **a2 = (Avlnode **)av; + unsigned short s; + + if (!av) + return NULL; + + memcpy(&s, val->bv_val, 2); + return avl_find( a2[s], val->bv_val+2, syncuuid_cmp ); +#else + return avl_find( av, val->bv_val, syncuuid_cmp ); +#endif +} + +static int +presentlist_free( Avlnode *av ) +{ +#ifdef HASHUUID + Avlnode **a2 = (Avlnode **)av; + int i, count = 0; + + if ( av ) { + for (i=0; i<65536; i++) { + if (a2[i]) + count += avl_free( a2[i], ch_free ); + } + ch_free( av ); + } + return count; +#else + return avl_free( av, ch_free ); +#endif +} + +static void +presentlist_delete( + Avlnode **av, + struct berval *val ) +{ +#ifdef HASHUUID + Avlnode **a2 = *(Avlnode ***)av; + unsigned short s; + + memcpy(&s, val->bv_val, 2); + avl_delete( &a2[s], val->bv_val+2, syncuuid_cmp ); +#else + avl_delete( av, val->bv_val, syncuuid_cmp ); +#endif +} + static int syncrepl_entry( syncinfo_t* si, @@ -2721,7 +2860,7 @@ syncrepl_entry( if (( syncstate == LDAP_SYNC_PRESENT || syncstate == LDAP_SYNC_ADD ) ) { if ( !si->si_refreshPresent && !si->si_refreshDone ) { - syncuuid_inserted = avl_presentlist_insert( si, syncUUID ); + syncuuid_inserted = presentlist_insert( si, syncUUID ); } } @@ -2830,6 +2969,23 @@ syncrepl_entry( slap_queue_csn( op, syncCSN ); } + if ( !si->si_refreshDone ) { + if ( si->si_lazyCommit ) + op->o_lazyCommit = SLAP_CONTROL_NONCRITICAL; + if ( si->si_refreshCount == 500 ) { + LDAP_SLIST_REMOVE( &op->o_extra, si->si_refreshTxn, OpExtra, oe_next ); + op->o_bd->bd_info->bi_op_txn( op, SLAP_TXN_COMMIT, &si->si_refreshTxn ); + si->si_refreshCount = 0; + si->si_refreshTxn = NULL; + } + if ( op->o_bd->bd_info->bi_op_txn ) { + if ( !si->si_refreshCount ) { + op->o_bd->bd_info->bi_op_txn( op, SLAP_TXN_BEGIN, &si->si_refreshTxn ); + } + si->si_refreshCount++; + } + } + slap_op_time( &op->o_time, &op->o_tincr ); switch ( syncstate ) { case LDAP_SYNC_ADD: @@ -2880,6 +3036,7 @@ retry_add:; /* Something's wrong, start over */ ber_bvarray_free( si->si_syncCookie.ctxcsn ); si->si_syncCookie.ctxcsn = NULL; + entry_free( entry ); ldap_pvt_thread_mutex_lock( &si->si_cookieState->cs_mutex ); ber_bvarray_free( si->si_cookieState->cs_vals ); ch_free( si->si_cookieState->cs_sids ); @@ -3819,12 +3976,6 @@ syncrepl_updateCookie( ch_free( sc.sids ); ber_bvarray_free( sc.ctxcsn ); } - ldap_pvt_thread_mutex_unlock( &si->si_cookieState->cs_mutex ); - - op->o_bd = be; - op->o_tmpfree( op->o_csn.bv_val, op->o_tmpmemctx ); - BER_BVZERO( &op->o_csn ); - if ( mod.sml_next ) slap_mods_free( mod.sml_next, 1 ); #ifdef CHECK_CSN for ( i=0; isi_cookieState->cs_num; i++ ) { @@ -3832,6 +3983,13 @@ syncrepl_updateCookie( } #endif + ldap_pvt_thread_mutex_unlock( &si->si_cookieState->cs_mutex ); + + op->o_bd = be; + op->o_tmpfree( op->o_csn.bv_val, op->o_tmpmemctx ); + BER_BVZERO( &op->o_csn ); + if ( mod.sml_next ) slap_mods_free( mod.sml_next, 1 ); + return rc; } @@ -4210,11 +4368,11 @@ nonpresent_callback( syncinfo_t *si = op->o_callback->sc_private; Attribute *a; int count = 0; - struct berval* present_uuid = NULL; + char *present_uuid = NULL; struct nonpresent_entry *np_entry; if ( rs->sr_type == REP_RESULT ) { - count = avl_free( si->si_presentlist, ch_free ); + count = presentlist_free( si->si_presentlist ); si->si_presentlist = NULL; } else if ( rs->sr_type == REP_SEARCH ) { @@ -4222,8 +4380,7 @@ nonpresent_callback( a = attr_find( rs->sr_entry->e_attrs, slap_schema.si_ad_entryUUID ); if ( a ) { - present_uuid = avl_find( si->si_presentlist, &a->a_nvals[0], - syncuuid_cmp ); + present_uuid = presentlist_find( si->si_presentlist, &a->a_nvals[0] ); } if ( LogTest( LDAP_DEBUG_SYNC ) ) { @@ -4247,8 +4404,7 @@ nonpresent_callback( LDAP_LIST_INSERT_HEAD( &si->si_nonpresentlist, np_entry, npe_link ); } else { - avl_delete( &si->si_presentlist, - &a->a_nvals[0], syncuuid_cmp ); + presentlist_delete( &si->si_presentlist, &a->a_nvals[0] ); ch_free( present_uuid ); } } @@ -4379,11 +4535,11 @@ done:; static int syncuuid_cmp( const void* v_uuid1, const void* v_uuid2 ) { - const struct berval *uuid1 = v_uuid1; - const struct berval *uuid2 = v_uuid2; - int rc = uuid1->bv_len - uuid2->bv_len; - if ( rc ) return rc; - return ( memcmp( uuid1->bv_val, uuid2->bv_val, uuid1->bv_len ) ); +#ifdef HASHUUID + return ( memcmp( v_uuid1, v_uuid2, UUIDLEN-2 )); +#else + return ( memcmp( v_uuid1, v_uuid2, UUIDLEN )); +#endif } void @@ -4481,7 +4637,7 @@ syncinfo_free( syncinfo_t *sie, int free_all ) } slap_sync_cookie_free( &sie->si_syncCookie, 0 ); if ( sie->si_presentlist ) { - avl_free( sie->si_presentlist, ch_free ); + presentlist_free( sie->si_presentlist ); } while ( !LDAP_LIST_EMPTY( &sie->si_nonpresentlist ) ) { struct nonpresent_entry* npe; @@ -4582,6 +4738,7 @@ config_suffixm( ConfigArgs *c, syncinfo_t *si ) #define LOGFILTERSTR "logfilter" #define SUFFIXMSTR "suffixmassage" #define STRICT_REFRESH "strictrefresh" +#define LAZY_COMMIT "lazycommit" /* FIXME: undocumented */ #define EXATTRSSTR "exattrs" @@ -5084,6 +5241,10 @@ parse_syncrepl_line( STRLENOF( STRICT_REFRESH ) ) ) { si->si_strict_refresh = 1; + } else if ( !strncasecmp( c->argv[ i ], LAZY_COMMIT, + STRLENOF( LAZY_COMMIT ) ) ) + { + si->si_lazyCommit = 1; } else if ( !bindconf_parse( c->argv[i], &si->si_bindconf ) ) { si->si_got |= GOT_BINDCONF; } else { @@ -5484,6 +5645,11 @@ syncrepl_unparse( syncinfo_t *si, struct berval *bv ) ptr = lutil_strcopy( ptr, bc.bv_val ); } } + + if ( si->si_lazyCommit ) { + ptr = lutil_strcopy( ptr, " " LAZY_COMMIT ); + } + bc.bv_len = ptr - buf; bc.bv_val = buf; ber_dupbv( bv, &bc );