X-Git-Url: https://git.sur5r.net/?a=blobdiff_plain;f=servers%2Fslapd%2Fsyncrepl.c;h=91f4d3a6b75e73bf4ea498e83e2e4cb4e7a320e0;hb=cff264c6e1b91632b1722506c022e519a216e422;hp=1b1db2f7373b4d4e7e4f5c36a4227106bd9ed106;hpb=ca15238dfbd58d46126eb2bd91bc987f03207cd4;p=openldap
diff --git a/servers/slapd/syncrepl.c b/servers/slapd/syncrepl.c
index 1b1db2f737..91f4d3a6b7 100644
--- a/servers/slapd/syncrepl.c
+++ b/servers/slapd/syncrepl.c
@@ -2,7 +2,7 @@
/* $OpenLDAP$ */
/* This work is part of OpenLDAP Software .
*
- * Copyright 2003-2011 The OpenLDAP Foundation.
+ * Copyright 2003-2017 The OpenLDAP Foundation.
* Portions Copyright 2003 by IBM Corporation.
* Portions Copyright 2003-2008 by Howard Chu, Symas Corporation.
* All rights reserved.
@@ -36,6 +36,8 @@
#define SUFFIXM_CTX ""
#endif
+#define UUIDLEN 16
+
struct nonpresent_entry {
struct berval *npe_name;
struct berval *npe_nname;
@@ -80,8 +82,9 @@ typedef struct syncinfo_s {
struct berval si_base;
struct berval si_logbase;
struct berval si_filterstr;
- Filter *si_filter;
struct berval si_logfilterstr;
+ Filter *si_filter;
+ Filter *si_logfilter;
struct berval si_contextdn;
int si_scope;
int si_attrsonly;
@@ -108,10 +111,16 @@ typedef struct syncinfo_s {
int si_refreshDelete;
int si_refreshPresent;
int si_refreshDone;
+ int si_refreshCount;
+ time_t si_refreshBeg;
+ time_t si_refreshEnd;
+ OpExtra *si_refreshTxn;
int si_syncdata;
int si_logstate;
+ int si_lazyCommit;
int si_got;
int si_strict_refresh; /* stop listening during fallback refresh */
+ int si_too_old;
ber_int_t si_msgid;
Avlnode *si_presentlist;
LDAP *si_ld;
@@ -125,13 +134,16 @@ typedef struct syncinfo_s {
} syncinfo_t;
static int syncuuid_cmp( const void *, const void * );
-static int avl_presentlist_insert( syncinfo_t* si, struct berval *syncUUID );
+static int presentlist_insert( syncinfo_t* si, struct berval *syncUUID );
+static void presentlist_delete( Avlnode **av, struct berval *syncUUID );
+static char *presentlist_find( Avlnode *av, struct berval *syncUUID );
+static int presentlist_free( Avlnode *av );
static void syncrepl_del_nonpresent( Operation *, syncinfo_t *, BerVarray, struct sync_cookie *, int );
static int syncrepl_message_to_op(
syncinfo_t *, Operation *, LDAPMessage * );
static int syncrepl_message_to_entry(
syncinfo_t *, Operation *, LDAPMessage *,
- Modifications **, Entry **, int );
+ Modifications **, Entry **, int, struct berval* );
static int syncrepl_entry(
syncinfo_t *, Operation*, Entry*,
Modifications**,int, struct berval*,
@@ -589,7 +601,8 @@ check_syncprov(
slap_compose_sync_cookie( NULL, &si->si_syncCookie.octet_str,
si->si_syncCookie.ctxcsn, si->si_syncCookie.rid,
si->si_syncCookie.sid );
- slap_parse_sync_cookie( &si->si_syncCookie, NULL );
+ ch_free( si->si_syncCookie.sids );
+ slap_reparse_sync_cookie( &si->si_syncCookie, op->o_tmpmemctx );
}
ldap_pvt_thread_mutex_unlock( &si->si_cookieState->cs_mutex );
return changed;
@@ -666,11 +679,11 @@ do_syncrep1(
LDAP_STAILQ_REMOVE( &slap_sync_cookie, sc, sync_cookie, sc_next );
- /* ctxcsn wasn't parsed yet, do it now */
- slap_parse_sync_cookie( sc, op->o_tmpmemctx );
slap_sync_cookie_free( &si->si_syncCookie, 0 );
- slap_dup_sync_cookie( &si->si_syncCookie, sc );
- slap_sync_cookie_free( sc, 1 );
+ si->si_syncCookie.octet_str = sc->octet_str;
+ ch_free( sc );
+ /* ctxcsn wasn't parsed yet, do it now */
+ slap_parse_sync_cookie( &si->si_syncCookie, NULL );
} else {
ldap_pvt_thread_mutex_lock( &si->si_cookieState->cs_mutex );
if ( !si->si_cookieState->cs_num ) {
@@ -711,6 +724,7 @@ do_syncrep1(
ldap_pvt_thread_mutex_unlock( &si->si_cookieState->cs_mutex );
}
+ ch_free( si->si_syncCookie.octet_str.bv_val );
slap_compose_sync_cookie( NULL, &si->si_syncCookie.octet_str,
si->si_syncCookie.ctxcsn, si->si_syncCookie.rid,
si->si_syncCookie.sid );
@@ -727,6 +741,11 @@ do_syncrep1(
}
si->si_refreshDone = 0;
+ si->si_refreshBeg = slap_get_time();
+ si->si_refreshCount = 0;
+ si->si_refreshTxn = NULL;
+ Debug( LDAP_DEBUG_ANY, "do_syncrep1: %s starting refresh\n",
+ si->si_ridtxt, 0, 0 );
rc = ldap_sync_search( si, op->o_tmpmemctx );
@@ -833,7 +852,7 @@ do_syncrep2(
tout_p, &msg ) ) > 0 )
{
int match, punlock, syncstate;
- struct berval *retdata, syncUUID, cookie = BER_BVNULL;
+ struct berval *retdata, syncUUID[2], cookie = BER_BVNULL;
char *retoid;
LDAPControl **rctrls = NULL, *rctrlp = NULL;
BerVarray syncUUIDs;
@@ -885,7 +904,7 @@ do_syncrep2(
goto done;
}
ber_init2( ber, &rctrlp->ldctl_value, LBER_USE_DER );
- if ( ber_scanf( ber, "{em" /*"}"*/, &syncstate, &syncUUID )
+ if ( ber_scanf( ber, "{em" /*"}"*/, &syncstate, &syncUUID[0] )
== LBER_ERROR ) {
bdn.bv_val[bdn.bv_len] = '\0';
Debug( LDAP_DEBUG_ANY, "do_syncrep2: %s malformed message (%s)\n",
@@ -896,10 +915,10 @@ do_syncrep2(
}
/* FIXME: what if syncUUID is NULL or empty?
* (happens with back-sql...) */
- if ( BER_BVISEMPTY( &syncUUID ) ) {
+ if ( syncUUID[0].bv_len != UUIDLEN ) {
bdn.bv_val[bdn.bv_len] = '\0';
Debug( LDAP_DEBUG_ANY, "do_syncrep2: %s "
- "got empty syncUUID with LDAP_SYNC_%s (%s)\n",
+ "got empty or invalid syncUUID with LDAP_SYNC_%s (%s)\n",
si->si_ridtxt,
syncrepl_state2str( syncstate ), bdn.bv_val );
ldap_controls_free( rctrls );
@@ -908,7 +927,7 @@ do_syncrep2(
}
punlock = -1;
if ( ber_peek_tag( ber, &len ) == LDAP_TAG_SYNC_COOKIE ) {
- ber_scanf( ber, /*"{"*/ "m}", &cookie );
+ if ( ber_scanf( ber, /*"{"*/ "m}", &cookie ) != LBER_ERROR ) {
Debug( LDAP_DEBUG_SYNC, "do_syncrep2: %s cookie=%s\n",
si->si_ridtxt,
@@ -924,7 +943,12 @@ do_syncrep2(
if ( syncCookie.ctxcsn ) {
int i, sid = slap_parse_csn_sid( syncCookie.ctxcsn );
check_syncprov( op, si );
+ ldap_pvt_thread_mutex_lock( &si->si_cookieState->cs_mutex );
for ( i =0; isi_cookieState->cs_num; i++ ) {
+#ifdef CHATTY_SYNCLOG
+ Debug( LDAP_DEBUG_SYNC, "do_syncrep2: %s CSN for sid %d: %s\n",
+ si->si_ridtxt, i, si->si_cookieState->cs_vals[i].bv_val );
+#endif
/* new SID */
if ( sid < si->si_cookieState->cs_sids[i] )
break;
@@ -933,13 +957,18 @@ do_syncrep2(
bdn.bv_val[bdn.bv_len] = '\0';
Debug( LDAP_DEBUG_SYNC, "do_syncrep2: %s CSN too old, ignoring %s (%s)\n",
si->si_ridtxt, syncCookie.ctxcsn->bv_val, bdn.bv_val );
+ si->si_too_old = 1;
+ ldap_pvt_thread_mutex_unlock( &si->si_cookieState->cs_mutex );
ldap_controls_free( rctrls );
rc = 0;
goto done;
}
+ si->si_too_old = 0;
break;
}
}
+ ldap_pvt_thread_mutex_unlock( &si->si_cookieState->cs_mutex );
+
/* check pending CSNs too */
while ( ldap_pvt_thread_mutex_trylock( &si->si_cookieState->cs_pmutex )) {
if ( slapd_shutdown ) {
@@ -949,17 +978,18 @@ do_syncrep2(
if ( !ldap_pvt_thread_pool_pausecheck( &connection_pool ))
ldap_pvt_thread_yield();
}
+
for ( i =0; isi_cookieState->cs_pnum; i++ ) {
if ( sid < si->si_cookieState->cs_psids[i] )
break;
if ( si->si_cookieState->cs_psids[i] == sid ) {
if ( ber_bvcmp( syncCookie.ctxcsn, &si->si_cookieState->cs_pvals[i] ) <= 0 ) {
+ ldap_pvt_thread_mutex_unlock( &si->si_cookieState->cs_pmutex );
bdn.bv_val[bdn.bv_len] = '\0';
Debug( LDAP_DEBUG_SYNC, "do_syncrep2: %s CSN pending, ignoring %s (%s)\n",
si->si_ridtxt, syncCookie.ctxcsn->bv_val, bdn.bv_val );
ldap_controls_free( rctrls );
rc = 0;
- ldap_pvt_thread_mutex_unlock( &si->si_cookieState->cs_pmutex );
goto done;
}
ber_bvreplace( &si->si_cookieState->cs_pvals[i],
@@ -976,9 +1006,17 @@ do_syncrep2(
}
assert( punlock < 0 );
punlock = i;
+ } else if (si->si_too_old) {
+ bdn.bv_val[bdn.bv_len] = '\0';
+ Debug( LDAP_DEBUG_SYNC, "do_syncrep2: %s CSN too old, ignoring (%s)\n",
+ si->si_ridtxt, bdn.bv_val, 0 );
+ ldap_controls_free( rctrls );
+ rc = 0;
+ goto done;
}
op->o_controls[slap_cids.sc_LDAPsync] = &syncCookie;
}
+ }
}
rc = 0;
if ( si->si_syncdata && si->si_logstate == SYNCLOG_LOGGING ) {
@@ -992,6 +1030,7 @@ do_syncrep2(
case LDAP_NO_SUCH_OBJECT:
case LDAP_NO_SUCH_ATTRIBUTE:
case LDAP_TYPE_OR_VALUE_EXISTS:
+ case LDAP_NOT_ALLOWED_ON_NONLEAF:
rc = LDAP_SYNC_REFRESH_REQUIRED;
si->si_logstate = SYNCLOG_FALLBACK;
ldap_abandon_ext( si->si_ld, si->si_msgid, NULL, NULL );
@@ -1007,10 +1046,10 @@ do_syncrep2(
break;
}
} else if ( ( rc = syncrepl_message_to_entry( si, op, msg,
- &modlist, &entry, syncstate ) ) == LDAP_SUCCESS )
+ &modlist, &entry, syncstate, syncUUID ) ) == LDAP_SUCCESS )
{
if ( ( rc = syncrepl_entry( si, op, entry, &modlist,
- syncstate, &syncUUID, syncCookie.ctxcsn ) ) == LDAP_SUCCESS &&
+ syncstate, syncUUID, syncCookie.ctxcsn ) ) == LDAP_SUCCESS &&
syncCookie.ctxcsn )
{
rc = syncrepl_updateCookie( si, op, &syncCookie );
@@ -1020,6 +1059,7 @@ do_syncrep2(
/* on failure, revert pending CSN */
if ( rc != LDAP_SUCCESS ) {
int i;
+ ldap_pvt_thread_mutex_lock( &si->si_cookieState->cs_mutex );
for ( i = 0; isi_cookieState->cs_num; i++ ) {
if ( si->si_cookieState->cs_sids[i] == si->si_cookieState->cs_psids[punlock] ) {
ber_bvreplace( &si->si_cookieState->cs_pvals[punlock],
@@ -1029,6 +1069,7 @@ do_syncrep2(
}
if ( i == si->si_cookieState->cs_num )
si->si_cookieState->cs_pvals[punlock].bv_val[0] = '\0';
+ ldap_pvt_thread_mutex_unlock( &si->si_cookieState->cs_mutex );
}
ldap_pvt_thread_mutex_unlock( &si->si_cookieState->cs_pmutex );
}
@@ -1150,8 +1191,8 @@ do_syncrep2(
{
syncrepl_del_nonpresent( op, si, NULL,
&syncCookie, m );
- } else {
- avl_free( si->si_presentlist, ch_free );
+ } else if ( si->si_presentlist ) {
+ presentlist_free( si->si_presentlist );
si->si_presentlist = NULL;
}
}
@@ -1159,6 +1200,13 @@ do_syncrep2(
{
rc = syncrepl_updateCookie( si, op, &syncCookie );
}
+ if ( si->si_refreshCount ) {
+ LDAP_SLIST_REMOVE( &op->o_extra, si->si_refreshTxn, OpExtra, oe_next );
+ op->o_bd->bd_info->bi_op_txn( op, SLAP_TXN_COMMIT, &si->si_refreshTxn );
+ si->si_refreshCount = 0;
+ si->si_refreshTxn = NULL;
+ }
+ si->si_refreshEnd = slap_get_time();
if ( err == LDAP_SUCCESS
&& si->si_logstate == SYNCLOG_FALLBACK ) {
si->si_logstate = SYNCLOG_LOGGING;
@@ -1240,6 +1288,17 @@ do_syncrep2(
{
si->si_refreshDone = 1;
}
+ if ( si->si_refreshDone ) {
+ if ( si->si_refreshCount ) {
+ LDAP_SLIST_REMOVE( &op->o_extra, si->si_refreshTxn, OpExtra, oe_next );
+ op->o_bd->bd_info->bi_op_txn( op, SLAP_TXN_COMMIT, &si->si_refreshTxn );
+ si->si_refreshCount = 0;
+ si->si_refreshTxn = NULL;
+ }
+ si->si_refreshEnd = slap_get_time();
+ Debug( LDAP_DEBUG_ANY, "do_syncrep1: %s finished refresh\n",
+ si->si_ridtxt, 0, 0 );
+ }
ber_scanf( ber, /*"{"*/ "}" );
if ( abs(si->si_type) == LDAP_SYNC_REFRESH_AND_PERSIST &&
si->si_refreshDone )
@@ -1278,20 +1337,23 @@ do_syncrep2(
ber_scanf( ber, "b", &refreshDeletes );
}
syncUUIDs = NULL;
- ber_scanf( ber, "[W]", &syncUUIDs );
+ rc = ber_scanf( ber, "[W]", &syncUUIDs );
ber_scanf( ber, /*"{"*/ "}" );
- if ( refreshDeletes ) {
- syncrepl_del_nonpresent( op, si, syncUUIDs,
- &syncCookie, m );
- ber_bvarray_free_x( syncUUIDs, op->o_tmpmemctx );
- } else {
- int i;
- for ( i = 0; !BER_BVISNULL( &syncUUIDs[i] ); i++ ) {
- (void)avl_presentlist_insert( si, &syncUUIDs[i] );
- slap_sl_free( syncUUIDs[i].bv_val, op->o_tmpmemctx );
+ if ( rc != LBER_ERROR ) {
+ if ( refreshDeletes ) {
+ syncrepl_del_nonpresent( op, si, syncUUIDs,
+ &syncCookie, m );
+ ber_bvarray_free_x( syncUUIDs, op->o_tmpmemctx );
+ } else {
+ int i;
+ for ( i = 0; !BER_BVISNULL( &syncUUIDs[i] ); i++ ) {
+ (void)presentlist_insert( si, &syncUUIDs[i] );
+ slap_sl_free( syncUUIDs[i].bv_val, op->o_tmpmemctx );
+ }
+ slap_sl_free( syncUUIDs, op->o_tmpmemctx );
}
- slap_sl_free( syncUUIDs, op->o_tmpmemctx );
}
+ rc = 0;
slap_sync_cookie_free( &syncCookie, 0 );
break;
default:
@@ -1328,11 +1390,18 @@ do_syncrep2(
{
rc = syncrepl_updateCookie( si, op, &syncCookie);
}
+ if ( si->si_presentlist ) {
+ presentlist_free( si->si_presentlist );
+ si->si_presentlist = NULL;
+ }
}
ldap_memfree( retoid );
ber_bvfree( retdata );
+ if ( rc )
+ goto done;
+
} else {
Debug( LDAP_DEBUG_ANY, "do_syncrep2: %s "
"unknown intermediate response (%d)\n",
@@ -1352,14 +1421,20 @@ do_syncrep2(
}
if ( !BER_BVISNULL( &syncCookie.octet_str ) ) {
slap_sync_cookie_free( &syncCookie_req, 0 );
- slap_dup_sync_cookie( &syncCookie_req, &syncCookie );
- slap_sync_cookie_free( &syncCookie, 0 );
+ syncCookie_req = syncCookie;
+ memset( &syncCookie, 0, sizeof( syncCookie ));
}
ldap_msgfree( msg );
msg = NULL;
if ( ldap_pvt_thread_pool_pausing( &connection_pool )) {
slap_sync_cookie_free( &syncCookie, 0 );
slap_sync_cookie_free( &syncCookie_req, 0 );
+ if ( si->si_refreshCount ) {
+ LDAP_SLIST_REMOVE( &op->o_extra, si->si_refreshTxn, OpExtra, oe_next );
+ op->o_bd->bd_info->bi_op_txn( op, SLAP_TXN_COMMIT, &si->si_refreshTxn );
+ si->si_refreshCount = 0;
+ si->si_refreshTxn = NULL;
+ }
return SYNC_PAUSED;
}
}
@@ -1425,6 +1500,8 @@ do_syncrepl(
ldap_pvt_thread_yield();
}
+ si->si_too_old = 0;
+
if ( si->si_ctype < 1 ) {
goto deleted;
}
@@ -1499,7 +1576,7 @@ do_syncrepl(
si->si_refreshPresent = 0;
if ( si->si_presentlist ) {
- avl_free( si->si_presentlist, ch_free );
+ presentlist_free( si->si_presentlist );
si->si_presentlist = NULL;
}
@@ -1521,6 +1598,10 @@ reload:
op->o_ndn = op->o_bd->be_rootndn;
rc = do_syncrep2( op, si );
if ( rc == LDAP_SYNC_REFRESH_REQUIRED ) {
+ if ( BER_BVISNULL( &si->si_syncCookie.octet_str ))
+ slap_compose_sync_cookie( NULL, &si->si_syncCookie.octet_str,
+ si->si_syncCookie.ctxcsn, si->si_syncCookie.rid,
+ si->si_syncCookie.sid );
rc = ldap_sync_search( si, op->o_tmpmemctx );
goto reload;
}
@@ -1714,6 +1795,11 @@ syncrepl_accesslog_mods(
if ( !colon ) {
/* Invalid */
continue;
+ } else if ( colon == bv.bv_val ) {
+ /* ITS#6545: An empty attribute signals that a new mod
+ * is about to start */
+ mod = NULL;
+ continue;
}
bv.bv_len = colon - bv.bv_val;
@@ -1961,6 +2047,7 @@ drop:
}
}
slap_mods_free( newmods, 1 );
+ rx->rx_mods = oldmods;
}
}
return LDAP_SUCCESS;
@@ -2021,6 +2108,33 @@ syncrepl_op_modify( Operation *op, SlapReply *rs )
if ( !mod )
return SLAP_CB_CONTINUE;
+ {
+ int i, sid;
+ sid = slap_parse_csn_sid( &mod->sml_nvalues[0] );
+ ldap_pvt_thread_mutex_lock( &si->si_cookieState->cs_mutex );
+ for ( i =0; isi_cookieState->cs_num; i++ ) {
+#ifdef CHATTY_SYNCLOG
+ Debug( LDAP_DEBUG_SYNC, "syncrepl_op_modify: %s CSN for sid %d: %s\n",
+ si->si_ridtxt, i, si->si_cookieState->cs_vals[i].bv_val );
+#endif
+ /* new SID */
+ if ( sid < si->si_cookieState->cs_sids[i] )
+ break;
+ if ( si->si_cookieState->cs_sids[i] == sid ) {
+ if ( ber_bvcmp( &mod->sml_nvalues[0], &si->si_cookieState->cs_vals[i] ) <= 0 ) {
+ Debug( LDAP_DEBUG_SYNC, "syncrepl_op_modify: %s entryCSN too old, ignoring %s (%s)\n",
+ si->si_ridtxt, mod->sml_nvalues[0].bv_val, op->o_req_dn.bv_val );
+ ldap_pvt_thread_mutex_unlock( &si->si_cookieState->cs_mutex );
+ slap_graduate_commit_csn( op );
+ /* tell accesslog this was a failure */
+ rs->sr_err = LDAP_TYPE_OR_VALUE_EXISTS;
+ return LDAP_SUCCESS;
+ }
+ }
+ }
+ ldap_pvt_thread_mutex_unlock( &si->si_cookieState->cs_mutex );
+ }
+
rc = overlay_entry_get_ov( op, &op->o_req_ndn, NULL, NULL, 0, &e, on );
if ( rc == 0 ) {
Attribute *a;
@@ -2033,8 +2147,12 @@ syncrepl_op_modify( Operation *op, SlapReply *rs )
overlay_entry_release_ov( op, e, 0, on );
}
/* equal? Should never happen */
- if ( match == 0 )
+ if ( match == 0 ) {
+ slap_graduate_commit_csn( op );
+ /* tell accesslog this was a failure */
+ rs->sr_err = LDAP_TYPE_OR_VALUE_EXISTS;
return LDAP_SUCCESS;
+ }
/* mod is older: resolve conflicts...
* 1. Save/copy original modlist. Split Replace to Del/Add.
@@ -2079,6 +2197,8 @@ syncrepl_op_modify( Operation *op, SlapReply *rs )
SlapReply rs1 = {0};
resolve_ctxt rx;
slap_callback cb = { NULL, syncrepl_resolve_cb, NULL, NULL };
+ Filter lf[3] = {0};
+ AttributeAssertion aa[2] = {0};
rx.rx_si = si;
rx.rx_mods = newlist;
@@ -2106,7 +2226,21 @@ syncrepl_op_modify( Operation *op, SlapReply *rs )
op2.ors_filterstr.bv_len = sprintf(op2.ors_filterstr.bv_val,
"(&(entryCSN>=%s)(reqDN=%s)%s)",
bv.bv_val, op->o_req_ndn.bv_val, si->si_logfilterstr.bv_val );
- op2.ors_filter = str2filter_x( op, op2.ors_filterstr.bv_val );
+
+ lf[0].f_choice = LDAP_FILTER_AND;
+ lf[0].f_and = lf+1;
+ lf[1].f_choice = LDAP_FILTER_GE;
+ lf[1].f_ava = aa;
+ lf[1].f_av_desc = slap_schema.si_ad_entryCSN;
+ lf[1].f_av_value = bv;
+ lf[1].f_next = lf+2;
+ lf[2].f_choice = LDAP_FILTER_EQUALITY;
+ lf[2].f_ava = aa+1;
+ lf[2].f_av_desc = ad_reqDN;
+ lf[2].f_av_value = op->o_req_ndn;
+ lf[2].f_next = si->si_logfilter;
+
+ op2.ors_filter = lf;
op2.o_callback = &cb;
op2.o_bd = select_backend( &op2.o_req_ndn, 1 );
@@ -2124,6 +2258,7 @@ syncrepl_op_modify( Operation *op, SlapReply *rs )
sc->sc_private = mx;
sc->sc_next = op->o_callback;
sc->sc_cleanup = NULL;
+ sc->sc_writewait = NULL;
op->o_callback = sc;
op->orm_no_opattrs = 1;
mx->mx_orig = op->orm_modlist;
@@ -2415,7 +2550,8 @@ syncrepl_message_to_entry(
LDAPMessage *msg,
Modifications **modlist,
Entry **entry,
- int syncstate
+ int syncstate,
+ struct berval *syncUUID
)
{
Entry *e = NULL;
@@ -2457,6 +2593,14 @@ syncrepl_message_to_entry(
return LDAP_OTHER;
}
+ /* syncUUID[0] is normalized UUID received over the wire
+ * syncUUID[1] is denormalized UUID, generated here
+ */
+ (void)slap_uuidstr_from_normalized( &syncUUID[1], &syncUUID[0], op->o_tmpmemctx );
+ Debug( LDAP_DEBUG_SYNC,
+ "syncrepl_message_to_entry: %s DN: %s, UUID: %s\n",
+ si->si_ridtxt, bdn.bv_val, syncUUID[1].bv_val );
+
if ( syncstate == LDAP_SYNC_PRESENT || syncstate == LDAP_SYNC_DELETE ) {
/* NOTE: this could be done even before decoding the DN,
* although encoding errors wouldn't be detected */
@@ -2636,34 +2780,111 @@ typedef struct dninfo {
int delOldRDN; /* Was old RDN deleted? */
Modifications **modlist; /* the modlist we received */
Modifications *mods; /* the modlist we compared */
- Attribute *oldNattr; /* old naming attr */
+ int oldNcount; /* #values of old naming attr */
AttributeDescription *oldDesc; /* for renames */
AttributeDescription *newDesc; /* for renames */
} dninfo;
+#define HASHUUID 1
+
/* return 1 if inserted, 0 otherwise */
static int
-avl_presentlist_insert(
+presentlist_insert(
syncinfo_t* si,
struct berval *syncUUID )
{
- struct berval *syncuuid_bv = ch_malloc( sizeof( struct berval ) + syncUUID->bv_len + 1 );
+ char *val;
+
+#ifdef HASHUUID
+ Avlnode **av;
+ unsigned short s;
+
+ if ( !si->si_presentlist )
+ si->si_presentlist = ch_calloc(65536, sizeof( Avlnode * ));
+
+ av = (Avlnode **)si->si_presentlist;
- syncuuid_bv->bv_len = syncUUID->bv_len;
- syncuuid_bv->bv_val = (char *)&syncuuid_bv[1];
- AC_MEMCPY( syncuuid_bv->bv_val, syncUUID->bv_val, syncUUID->bv_len );
- syncuuid_bv->bv_val[ syncuuid_bv->bv_len ] = '\0';
+ val = ch_malloc(UUIDLEN-2);
+ memcpy(&s, syncUUID->bv_val, 2);
+ memcpy(val, syncUUID->bv_val+2, UUIDLEN-2);
- if ( avl_insert( &si->si_presentlist, (caddr_t) syncuuid_bv,
+ if ( avl_insert( &av[s], val,
syncuuid_cmp, avl_dup_error ) )
{
- ch_free( syncuuid_bv );
+ ch_free( val );
return 0;
}
+#else
+ val = ch_malloc(UUIDLEN);
+
+ AC_MEMCPY( val, syncUUID->bv_val, UUIDLEN );
+
+ if ( avl_insert( &si->si_presentlist, val,
+ syncuuid_cmp, avl_dup_error ) )
+ {
+ ch_free( val );
+ return 0;
+ }
+#endif
return 1;
}
+static char *
+presentlist_find(
+ Avlnode *av,
+ struct berval *val )
+{
+#ifdef HASHUUID
+ Avlnode **a2 = (Avlnode **)av;
+ unsigned short s;
+
+ if (!av)
+ return NULL;
+
+ memcpy(&s, val->bv_val, 2);
+ return avl_find( a2[s], val->bv_val+2, syncuuid_cmp );
+#else
+ return avl_find( av, val->bv_val, syncuuid_cmp );
+#endif
+}
+
+static int
+presentlist_free( Avlnode *av )
+{
+#ifdef HASHUUID
+ Avlnode **a2 = (Avlnode **)av;
+ int i, count = 0;
+
+ if ( av ) {
+ for (i=0; i<65536; i++) {
+ if (a2[i])
+ count += avl_free( a2[i], ch_free );
+ }
+ ch_free( av );
+ }
+ return count;
+#else
+ return avl_free( av, ch_free );
+#endif
+}
+
+static void
+presentlist_delete(
+ Avlnode **av,
+ struct berval *val )
+{
+#ifdef HASHUUID
+ Avlnode **a2 = *(Avlnode ***)av;
+ unsigned short s;
+
+ memcpy(&s, val->bv_val, 2);
+ avl_delete( &a2[s], val->bv_val+2, syncuuid_cmp );
+#else
+ avl_delete( av, val->bv_val, syncuuid_cmp );
+#endif
+}
+
static int
syncrepl_entry(
syncinfo_t* si,
@@ -2677,7 +2898,6 @@ syncrepl_entry(
Backend *be = op->o_bd;
slap_callback cb = { NULL, NULL, NULL, NULL };
int syncuuid_inserted = 0;
- struct berval syncUUID_strrep = BER_BVNULL;
SlapReply rs_search = {REP_RESULT};
Filter f = {0};
@@ -2695,7 +2915,7 @@ syncrepl_entry(
if (( syncstate == LDAP_SYNC_PRESENT || syncstate == LDAP_SYNC_ADD ) ) {
if ( !si->si_refreshPresent && !si->si_refreshDone ) {
- syncuuid_inserted = avl_presentlist_insert( si, syncUUID );
+ syncuuid_inserted = presentlist_insert( si, syncUUID );
}
}
@@ -2707,14 +2927,13 @@ syncrepl_entry(
}
}
- (void)slap_uuidstr_from_normalized( &syncUUID_strrep, syncUUID, op->o_tmpmemctx );
if ( syncstate != LDAP_SYNC_DELETE ) {
Attribute *a = attr_find( entry->e_attrs, slap_schema.si_ad_entryUUID );
if ( a == NULL ) {
/* add if missing */
attr_merge_one( entry, slap_schema.si_ad_entryUUID,
- &syncUUID_strrep, syncUUID );
+ &syncUUID[1], syncUUID );
} else if ( !bvmatch( &a->a_nvals[0], syncUUID ) ) {
/* replace only if necessary */
@@ -2723,7 +2942,7 @@ syncrepl_entry(
ber_dupbv( &a->a_nvals[0], syncUUID );
}
ber_memfree( a->a_vals[0].bv_val );
- ber_dupbv( &a->a_vals[0], &syncUUID_strrep );
+ ber_dupbv( &a->a_vals[0], &syncUUID[1] );
}
}
@@ -2734,16 +2953,16 @@ syncrepl_entry(
if ( syncuuid_inserted ) {
Debug( LDAP_DEBUG_SYNC, "syncrepl_entry: %s inserted UUID %s\n",
- si->si_ridtxt, syncUUID_strrep.bv_val, 0 );
+ si->si_ridtxt, syncUUID[1].bv_val, 0 );
}
op->ors_filter = &f;
- op->ors_filterstr.bv_len = STRLENOF( "(entryUUID=)" ) + syncUUID_strrep.bv_len;
+ op->ors_filterstr.bv_len = STRLENOF( "(entryUUID=)" ) + syncUUID[1].bv_len;
op->ors_filterstr.bv_val = (char *) slap_sl_malloc(
op->ors_filterstr.bv_len + 1, op->o_tmpmemctx );
AC_MEMCPY( op->ors_filterstr.bv_val, "(entryUUID=", STRLENOF( "(entryUUID=" ) );
AC_MEMCPY( &op->ors_filterstr.bv_val[STRLENOF( "(entryUUID=" )],
- syncUUID_strrep.bv_val, syncUUID_strrep.bv_len );
+ syncUUID[1].bv_val, syncUUID[1].bv_len );
op->ors_filterstr.bv_val[op->ors_filterstr.bv_len - 1] = ')';
op->ors_filterstr.bv_val[op->ors_filterstr.bv_len] = '\0';
@@ -2805,6 +3024,23 @@ syncrepl_entry(
slap_queue_csn( op, syncCSN );
}
+ if ( !si->si_refreshDone ) {
+ if ( si->si_lazyCommit )
+ op->o_lazyCommit = SLAP_CONTROL_NONCRITICAL;
+ if ( si->si_refreshCount == 500 ) {
+ LDAP_SLIST_REMOVE( &op->o_extra, si->si_refreshTxn, OpExtra, oe_next );
+ op->o_bd->bd_info->bi_op_txn( op, SLAP_TXN_COMMIT, &si->si_refreshTxn );
+ si->si_refreshCount = 0;
+ si->si_refreshTxn = NULL;
+ }
+ if ( op->o_bd->bd_info->bi_op_txn ) {
+ if ( !si->si_refreshCount ) {
+ op->o_bd->bd_info->bi_op_txn( op, SLAP_TXN_BEGIN, &si->si_refreshTxn );
+ }
+ si->si_refreshCount++;
+ }
+ }
+
slap_op_time( &op->o_time, &op->o_tincr );
switch ( syncstate ) {
case LDAP_SYNC_ADD:
@@ -2855,6 +3091,7 @@ retry_add:;
/* Something's wrong, start over */
ber_bvarray_free( si->si_syncCookie.ctxcsn );
si->si_syncCookie.ctxcsn = NULL;
+ entry_free( entry );
ldap_pvt_thread_mutex_lock( &si->si_cookieState->cs_mutex );
ber_bvarray_free( si->si_cookieState->cs_vals );
ch_free( si->si_cookieState->cs_sids );
@@ -2954,10 +3191,10 @@ retry_add:;
* If delOldRDN is TRUE then we should see a delete modop
* for oldDesc. We might see a replace instead.
* delete with no values: therefore newDesc != oldDesc.
- * if oldNattr had only one value, then Drop this op.
+ * if oldNcount == 1, then Drop this op.
* delete with 1 value: can only be the oldRDN value. Drop op.
* delete with N values: Drop oldRDN value, keep remainder.
- * replace with 1 value: if oldNattr had only one value and
+ * replace with 1 value: if oldNcount == 1 and
* newDesc == oldDesc, Drop this op.
* Any other cases must be left intact.
*
@@ -2978,7 +3215,7 @@ retry_add:;
continue;
}
if ( mod->sml_numvals <= 1 &&
- dni.oldNattr->a_numvals == 1 &&
+ dni.oldNcount == 1 &&
( mod->sml_op == LDAP_MOD_DELETE ||
mod->sml_op == LDAP_MOD_REPLACE )) {
if ( mod->sml_op == LDAP_MOD_REPLACE )
@@ -3181,6 +3418,8 @@ retry_modrdn:;
Debug( LDAP_DEBUG_SYNC,
"syncrepl_entry: %s be_delete %s (%d)\n",
si->si_ridtxt, op->o_req_dn.bv_val, rc );
+ if ( rc == LDAP_NO_SUCH_OBJECT )
+ rc = LDAP_SUCCESS;
while ( rs_delete.sr_err == LDAP_SUCCESS
&& op->o_delete_glue_parent ) {
@@ -3210,10 +3449,8 @@ retry_modrdn:;
}
done:
- if ( !BER_BVISNULL( &syncUUID_strrep ) ) {
- slap_sl_free( syncUUID_strrep.bv_val, op->o_tmpmemctx );
- BER_BVZERO( &syncUUID_strrep );
- }
+ slap_sl_free( syncUUID[1].bv_val, op->o_tmpmemctx );
+ BER_BVZERO( &syncUUID[1] );
if ( !BER_BVISNULL( &dni.ndn ) ) {
op->o_tmpfree( dni.ndn.bv_val, op->o_tmpmemctx );
}
@@ -3794,12 +4031,6 @@ syncrepl_updateCookie(
ch_free( sc.sids );
ber_bvarray_free( sc.ctxcsn );
}
- ldap_pvt_thread_mutex_unlock( &si->si_cookieState->cs_mutex );
-
- op->o_bd = be;
- op->o_tmpfree( op->o_csn.bv_val, op->o_tmpmemctx );
- BER_BVZERO( &op->o_csn );
- if ( mod.sml_next ) slap_mods_free( mod.sml_next, 1 );
#ifdef CHECK_CSN
for ( i=0; isi_cookieState->cs_num; i++ ) {
@@ -3807,6 +4038,13 @@ syncrepl_updateCookie(
}
#endif
+ ldap_pvt_thread_mutex_unlock( &si->si_cookieState->cs_mutex );
+
+ op->o_bd = be;
+ op->o_tmpfree( op->o_csn.bv_val, op->o_tmpmemctx );
+ BER_BVZERO( &op->o_csn );
+ if ( mod.sml_next ) slap_mods_free( mod.sml_next, 1 );
+
return rc;
}
@@ -4133,7 +4371,10 @@ dn_callback(
dni->oldDesc = ad;
for ( oldpos=0, a=rs->sr_entry->e_attrs;
a && a->a_desc != ad; oldpos++, a=a->a_next );
- dni->oldNattr = a;
+ /* a should not be NULL but apparently it happens.
+ * ITS#7144
+ */
+ dni->oldNcount = a ? a->a_numvals : 0;
for ( newpos=0, a=dni->new_entry->e_attrs;
a && a->a_desc != ad; newpos++, a=a->a_next );
if ( !a || oldpos != newpos || attr_valfind( a,
@@ -4182,11 +4423,11 @@ nonpresent_callback(
syncinfo_t *si = op->o_callback->sc_private;
Attribute *a;
int count = 0;
- struct berval* present_uuid = NULL;
+ char *present_uuid = NULL;
struct nonpresent_entry *np_entry;
if ( rs->sr_type == REP_RESULT ) {
- count = avl_free( si->si_presentlist, ch_free );
+ count = presentlist_free( si->si_presentlist );
si->si_presentlist = NULL;
} else if ( rs->sr_type == REP_SEARCH ) {
@@ -4194,8 +4435,7 @@ nonpresent_callback(
a = attr_find( rs->sr_entry->e_attrs, slap_schema.si_ad_entryUUID );
if ( a ) {
- present_uuid = avl_find( si->si_presentlist, &a->a_nvals[0],
- syncuuid_cmp );
+ present_uuid = presentlist_find( si->si_presentlist, &a->a_nvals[0] );
}
if ( LogTest( LDAP_DEBUG_SYNC ) ) {
@@ -4219,8 +4459,7 @@ nonpresent_callback(
LDAP_LIST_INSERT_HEAD( &si->si_nonpresentlist, np_entry, npe_link );
} else {
- avl_delete( &si->si_presentlist,
- &a->a_nvals[0], syncuuid_cmp );
+ presentlist_delete( &si->si_presentlist, &a->a_nvals[0] );
ch_free( present_uuid );
}
}
@@ -4351,11 +4590,11 @@ done:;
static int
syncuuid_cmp( const void* v_uuid1, const void* v_uuid2 )
{
- const struct berval *uuid1 = v_uuid1;
- const struct berval *uuid2 = v_uuid2;
- int rc = uuid1->bv_len - uuid2->bv_len;
- if ( rc ) return rc;
- return ( memcmp( uuid1->bv_val, uuid2->bv_val, uuid1->bv_len ) );
+#ifdef HASHUUID
+ return ( memcmp( v_uuid1, v_uuid2, UUIDLEN-2 ));
+#else
+ return ( memcmp( v_uuid1, v_uuid2, UUIDLEN ));
+#endif
}
void
@@ -4401,6 +4640,9 @@ syncinfo_free( syncinfo_t *sie, int free_all )
if ( sie->si_logfilterstr.bv_val ) {
ch_free( sie->si_logfilterstr.bv_val );
}
+ if ( sie->si_logfilter ) {
+ filter_free( sie->si_logfilter );
+ }
if ( sie->si_base.bv_val ) {
ch_free( sie->si_base.bv_val );
}
@@ -4453,7 +4695,7 @@ syncinfo_free( syncinfo_t *sie, int free_all )
}
slap_sync_cookie_free( &sie->si_syncCookie, 0 );
if ( sie->si_presentlist ) {
- avl_free( sie->si_presentlist, ch_free );
+ presentlist_free( sie->si_presentlist );
}
while ( !LDAP_LIST_EMPTY( &sie->si_nonpresentlist ) ) {
struct nonpresent_entry* npe;
@@ -4554,6 +4796,7 @@ config_suffixm( ConfigArgs *c, syncinfo_t *si )
#define LOGFILTERSTR "logfilter"
#define SUFFIXMSTR "suffixmassage"
#define STRICT_REFRESH "strictrefresh"
+#define LAZY_COMMIT "lazycommit"
/* FIXME: undocumented */
#define EXATTRSSTR "exattrs"
@@ -5056,14 +5299,19 @@ parse_syncrepl_line(
STRLENOF( STRICT_REFRESH ) ) )
{
si->si_strict_refresh = 1;
- } else if ( bindconf_parse( c->argv[i], &si->si_bindconf ) ) {
+ } else if ( !strncasecmp( c->argv[ i ], LAZY_COMMIT,
+ STRLENOF( LAZY_COMMIT ) ) )
+ {
+ si->si_lazyCommit = 1;
+ } else if ( !bindconf_parse( c->argv[i], &si->si_bindconf ) ) {
+ si->si_got |= GOT_BINDCONF;
+ } else {
snprintf( c->cr_msg, sizeof( c->cr_msg ),
"Error: parse_syncrepl_line: "
"unable to parse \"%s\"\n", c->argv[ i ] );
Debug( LDAP_DEBUG_ANY, "%s: %s.\n", c->log, c->cr_msg, 0 );
return -1;
}
- si->si_got |= GOT_BINDCONF;
}
if ( ( si->si_got & GOT_REQUIRED ) != GOT_REQUIRED ) {
@@ -5077,11 +5325,11 @@ parse_syncrepl_line(
}
if ( !be_issubordinate( c->be, &si->si_base ) && !( si->si_got & GOT_SUFFIXM )) {
- ch_free( si->si_base.bv_val );
- BER_BVZERO( &si->si_base );
snprintf( c->cr_msg, sizeof( c->cr_msg ),
"Base DN \"%s\" is not within the database naming context",
- val );
+ si->si_base.bv_val );
+ ch_free( si->si_base.bv_val );
+ BER_BVZERO( &si->si_base );
Debug( LDAP_DEBUG_ANY, "%s: %s.\n", c->log, c->cr_msg, 0 );
return -1;
}
@@ -5116,6 +5364,15 @@ parse_syncrepl_line(
return 1;
}
+ if ( si->si_got & GOT_LOGFILTER ) {
+ si->si_logfilter = str2filter( si->si_logfilterstr.bv_val );
+ if ( si->si_logfilter == NULL ) {
+ Debug( LDAP_DEBUG_ANY, "syncrepl %s " SEARCHBASESTR "=\"%s\": unable to parse logfilter=\"%s\"\n",
+ si->si_ridtxt, c->be->be_suffix ? c->be->be_suffix[ 0 ].bv_val : "(null)", si->si_logfilterstr.bv_val );
+ return 1;
+ }
+ }
+
return 0;
}
@@ -5455,6 +5712,11 @@ syncrepl_unparse( syncinfo_t *si, struct berval *bv )
ptr = lutil_strcopy( ptr, bc.bv_val );
}
}
+
+ if ( si->si_lazyCommit ) {
+ ptr = lutil_strcopy( ptr, " " LAZY_COMMIT );
+ }
+
bc.bv_len = ptr - buf;
bc.bv_val = buf;
ber_dupbv( bv, &bc );
@@ -5492,7 +5754,7 @@ syncrepl_config( ConfigArgs *c )
* happen when running on the cn=config DB.
*/
if ( si->si_re ) {
- if ( ldap_pvt_thread_mutex_trylock( &si->si_mutex )) {
+ if ( si->si_be == c->be || ldap_pvt_thread_mutex_trylock( &si->si_mutex )) {
isrunning = 1;
} else {
/* There is no active thread, but we must still
@@ -5512,13 +5774,12 @@ syncrepl_config( ConfigArgs *c )
ldap_pvt_runqueue_stoptask( &slapd_rq, re );
isrunning = 1;
}
+ if ( ldap_pvt_thread_pool_retract( re->pool_cookie ) > 0 )
+ isrunning = 0;
+
ldap_pvt_runqueue_remove( &slapd_rq, re );
ldap_pvt_thread_mutex_unlock( &slapd_rq.rq_mutex );
- if ( ldap_pvt_thread_pool_retract( &connection_pool,
- re->routine, re ) > 0 )
- isrunning = 0;
-
ldap_pvt_thread_mutex_unlock( &si->si_mutex );
}
}