From: Howard Chu Date: Mon, 5 Feb 2007 04:50:07 +0000 (+0000) Subject: Preliminary multi-context support for syncrepl. Passes all single-master X-Git-Tag: OPENLDAP_REL_ENG_2_4_4ALPHA~8^2~88 X-Git-Url: https://git.sur5r.net/?a=commitdiff_plain;h=fffaea79eb73c79282488d549280386364048347;p=openldap Preliminary multi-context support for syncrepl. Passes all single-master tests, needs multi-master testing. --- diff --git a/servers/slapd/bconfig.c b/servers/slapd/bconfig.c index 3a0373babb..9ec4b7f10b 100644 --- a/servers/slapd/bconfig.c +++ b/servers/slapd/bconfig.c @@ -555,7 +555,7 @@ static ConfigTable config_back_cf_table[] = { "SYNTAX OMsDN )", NULL, NULL }, { "syncrepl", NULL, 0, 0, 0, ARG_DB|ARG_MAGIC, &syncrepl_config, "( OLcfgDbAt:0.11 NAME 'olcSyncrepl' " - "SYNTAX OMsDirectoryString SINGLE-VALUE )", NULL, NULL }, + "SYNTAX OMsDirectoryString X-ORDERED 'VALUES' )", NULL, NULL }, { "threads", "count", 2, 2, 0, #ifdef NO_THREADS ARG_IGNORED, NULL, @@ -760,8 +760,6 @@ typedef struct ServerID { int si_num; } ServerID; -#define SERVERID_MAX 4095 - static ServerID *sid_list; static int @@ -1574,7 +1572,7 @@ config_generic(ConfigArgs *c) { ServerID *si, **sip; LDAPURLDesc *lud; int num = atoi( c->argv[1] ); - if ( num < 0 || num > SERVERID_MAX ) { + if ( num < 0 || num > SLAP_SYNC_SID_MAX ) { snprintf( c->msg, sizeof( c->msg ), "<%s> illegal server ID", c->argv[0] ); Debug(LDAP_DEBUG_ANY, "%s: %s %s\n", @@ -1636,7 +1634,7 @@ config_generic(ConfigArgs *c) { for ( i=0; l[i]; i++ ) { LDAPURLDesc *lu2; int isMe = 0; - ldap_url_parse( &l[i]->sl_url, &lu2 ); + ldap_url_parse( l[i]->sl_url.bv_val, &lu2 ); do { if ( strcasecmp( lud->lud_scheme, lu2->lud_scheme )) diff --git a/servers/slapd/ldapsync.c b/servers/slapd/ldapsync.c index f1e6f318df..652622d0c9 100644 --- a/servers/slapd/ldapsync.c +++ b/servers/slapd/ldapsync.c @@ -34,34 +34,47 @@ void slap_compose_sync_cookie( Operation *op, struct berval *cookie, - struct berval *csn, + BerVarray csn, int rid ) { - char cookiestr[ LDAP_LUTIL_CSNSTR_BUFSIZE + 20 ]; - int len; + int len, numcsn = 0; - if ( BER_BVISNULL( csn )) { + if ( csn ) { + for (; !BER_BVISEMPTY( &csn[numcsn] ); numcsn++); + } + + if ( numcsn == 0 || rid == -1 ) { + char cookiestr[ LDAP_LUTIL_CSNSTR_BUFSIZE + 20 ]; if ( rid == -1 ) { cookiestr[0] = '\0'; len = 0; } else { - len = snprintf( cookiestr, LDAP_LUTIL_CSNSTR_BUFSIZE + 20, + len = snprintf( cookiestr, sizeof( cookiestr ), "rid=%03d", rid ); } + ber_str2bv_x( cookiestr, len, 1, cookie, + op ? op->o_tmpmemctx : NULL ); } else { - char *end = cookiestr + sizeof(cookiestr); - char *ptr = lutil_strcopy( cookiestr, "csn=" ); - len = csn->bv_len; - if ( ptr + len >= end ) - len = end - ptr; - ptr = lutil_strncopy( ptr, csn->bv_val, len ); - if ( rid != -1 && ptr < end - STRLENOF(",rid=xxx") ) { - ptr += sprintf( ptr, ",rid=%03d", rid ); + char *ptr; + int i; + + len = 0; + for ( i=0; ibv_val = slap_sl_malloc( len, op ? op->o_tmpmemctx : NULL ); + + len = sprintf( cookie->bv_val, "rid=%03d,csn=", rid ); + ptr = cookie->bv_val + len; + for ( i=0; ibv_val, csn->bv_len ); + *ptr++ = ';'; } - len = ptr - cookiestr; + ptr--; + *ptr = '\0'; + cookie->bv_len = ptr - cookie->bv_val; } - ber_str2bv_x( cookiestr, len, 1, cookie, - op ? op->o_tmpmemctx : NULL ); } void @@ -73,11 +86,16 @@ slap_sync_cookie_free( if ( cookie == NULL ) return; - if ( !BER_BVISNULL( &cookie->ctxcsn )) { - ch_free( cookie->ctxcsn.bv_val ); - BER_BVZERO( &cookie->ctxcsn ); + if ( cookie->sids ) { + ch_free( cookie->sids ); + cookie->sids = NULL; } + if ( cookie->ctxcsn ) { + ber_bvarray_free( cookie->ctxcsn ); + cookie->ctxcsn = NULL; + } + cookie->numcsns = 0; if ( !BER_BVISNULL( &cookie->octet_str )) { ch_free( cookie->octet_str.bv_val ); BER_BVZERO( &cookie->octet_str ); @@ -90,6 +108,37 @@ slap_sync_cookie_free( return; } +int +slap_parse_csn_sid( struct berval *csn ) +{ + char *p, *q; + int i; + + p = memchr( csn->bv_val, '#', csn->bv_len ); + if ( p ) + p = strchr( p+1, '#' ); + if ( !p ) + return -1; + p++; + i = strtoul( p, &q, 10 ); + if ( p == q || i > SLAP_SYNC_SID_MAX ) + i = -1; + return i; +} + +int * +slap_parse_csn_sids( BerVarray csns, int numcsns ) +{ + int i, *ret; + char *p, *q; + + ret = ch_malloc( numcsns * sizeof(int) ); + for ( i=0; irid = -1; - /* FIXME: may read past end of cookie->octet_str.bv_val */ - rid_ptr = strstr( cookie->octet_str.bv_val, "rid=" ); - if ( rid_ptr == NULL - || rid_ptr > &cookie->octet_str.bv_val[ cookie->octet_str.bv_len - STRLENOF( "rid=" ) ] ) - { - return -1; - } - - if ( rid_ptr[ STRLENOF( "rid=" ) ] == '-' ) { - return -1; - } - cookie->rid = strtoul( &rid_ptr[ STRLENOF( "rid=" ) ], &next, 10 ); - if ( next == &rid_ptr[ STRLENOF( "rid=" ) ] || ( next[ 0 ] != ',' && next[ 0 ] != '\0' ) ) { - return -1; - } - - while (( csn_ptr = strstr( cookie->octet_str.bv_val, "csn=" )) != NULL ) { - AttributeDescription *ad = slap_schema.si_ad_modifyTimestamp; - slap_syntax_validate_func *validate; - struct berval stamp; - - /* This only happens when called from main */ - if ( ad == NULL ) - break; - - if ( csn_ptr >= &cookie->octet_str.bv_val[ cookie->octet_str.bv_len - STRLENOF( "csn=" ) ] ) { - return -1; + cookie->ctxcsn = NULL; + cookie->sids = NULL; + cookie->numcsns = 0; + + end = cookie->octet_str.bv_val + cookie->octet_str.bv_len; + + for ( next=cookie->octet_str.bv_val; next < end; ) { + if ( !strncmp( next, "rid=", STRLENOF("rid=") )) { + rid_ptr = next; + cookie->rid = strtoul( &rid_ptr[ STRLENOF( "rid=" ) ], &next, 10 ); + if ( next == rid_ptr || next > end || *next != ',' ) { + return -1; + } + if ( *next == ',' ) { + next++; + } + if ( !ad ) { + break; + } + continue; } - - csn_str = csn_ptr + STRLENOF("csn="); - cval = strchr( csn_str, ',' ); - if ( cval && cval < &cookie->octet_str.bv_val[ cookie->octet_str.bv_len ] ) - csn_str_len = cval - csn_str; - else - csn_str_len = 0; - - /* FIXME use csnValidate when it gets implemented */ - csn_ptr = strchr( csn_str, '#' ); - if ( !csn_ptr || csn_str >= &cookie->octet_str.bv_val[ cookie->octet_str.bv_len ] ) break; - - stamp.bv_val = csn_str; - stamp.bv_len = csn_ptr - csn_str; - validate = ad->ad_type->sat_syntax->ssyn_validate; - if ( validate( ad->ad_type->sat_syntax, &stamp ) != LDAP_SUCCESS ) - break; - valid = 1; - break; + if ( !strncmp( next, "csn=", STRLENOF("csn=") )) { + slap_syntax_validate_func *validate; + struct berval stamp; + + csn_str = next + STRLENOF("csn="); + while ( next < end ) { + /* FIXME use csnValidate when it gets implemented */ + csn_ptr = strchr( csn_str, '#' ); + if ( !csn_ptr || csn_ptr > end ) + break; + /* ad will be NULL when called from main. we just + * want to parse the rid then. But we still iterate + * through the string to find the end. + */ + if ( ad ) { + stamp.bv_val = csn_str; + stamp.bv_len = csn_ptr - csn_str; + validate = ad->ad_type->sat_syntax->ssyn_validate; + if ( validate( ad->ad_type->sat_syntax, &stamp ) + != LDAP_SUCCESS ) + break; + } + cval = strchr( csn_ptr, ';' ); + if ( !cval ) + cval = strchr(csn_ptr, ',' ); + if ( cval ) + stamp.bv_len = cval - csn_str; + else + stamp.bv_len = end - csn_str; + if ( ad ) { + value_add_one( &cookie->ctxcsn, &stamp ); + cookie->numcsns++; + } + if ( cval ) { + next = cval + 1; + if ( *cval != ';' ) + break; + } else { + next = end; + break; + } + } + continue; + } + next++; } - if ( valid ) { - ber_str2bv_x( csn_str, csn_str_len, 1, &cookie->ctxcsn, memctx ); - } else { - BER_BVZERO( &cookie->ctxcsn ); + if ( cookie->numcsns ) { + cookie->sids = slap_parse_csn_sids( cookie->ctxcsn, cookie->numcsns ); } - return 0; } @@ -189,7 +254,9 @@ slap_init_sync_cookie_ctxcsn( ctxcsn.bv_val = octet_str.bv_val + 4; ctxcsn.bv_len = octet_str.bv_len - 4; - ber_dupbv( &cookie->ctxcsn, &ctxcsn ); + cookie->ctxcsn = NULL; + value_add_one( &cookie->ctxcsn, &ctxcsn ); + cookie->numcsns = 1; return 0; } @@ -201,14 +268,16 @@ slap_dup_sync_cookie( ) { struct sync_cookie *new; + int i; if ( src == NULL ) return NULL; if ( dst ) { - ch_free( dst->ctxcsn.bv_val ); + ber_bvarray_free( dst->ctxcsn ); + dst->ctxcsn = NULL; + dst->sids = NULL; ch_free( dst->octet_str.bv_val ); - BER_BVZERO( &dst->ctxcsn ); BER_BVZERO( &dst->octet_str ); new = dst; } else { @@ -217,9 +286,18 @@ slap_dup_sync_cookie( } new->rid = src->rid; - - if ( !BER_BVISNULL( &src->ctxcsn )) { - ber_dupbv( &new->ctxcsn, &src->ctxcsn ); + new->numcsns = src->numcsns; + + if ( src->numcsns ) { + if ( ber_bvarray_dup_x( &new->ctxcsn, src->ctxcsn, NULL )) { + if ( !dst ) { + ch_free( new ); + } + return NULL; + } + new->sids = ch_malloc( src->numcsns * sizeof(int) ); + for (i=0; inumcsns; i++) + new->sids[i] = src->sids[i]; } if ( !BER_BVISNULL( &src->octet_str )) { diff --git a/servers/slapd/overlays/syncprov.c b/servers/slapd/overlays/syncprov.c index cda82aa0fb..c7ad1417b8 100644 --- a/servers/slapd/overlays/syncprov.c +++ b/servers/slapd/overlays/syncprov.c @@ -103,6 +103,7 @@ typedef struct slog_entry { struct slog_entry *se_next; struct berval se_uuid; struct berval se_csn; + int se_sid; ber_tag_t se_tag; } slog_entry; @@ -118,7 +119,9 @@ typedef struct sessionlog { /* The main state for this overlay */ typedef struct syncprov_info_t { syncops *si_ops; - struct berval si_ctxcsn; /* ldapsync context */ + BerVarray si_ctxcsn; /* ldapsync context */ + int *si_sids; + int si_numcsns; int si_chkops; /* checkpointing info */ int si_chktime; int si_numops; /* number of ops since last checkpoint */ @@ -130,7 +133,6 @@ typedef struct syncprov_info_t { ldap_pvt_thread_mutex_t si_csn_mutex; ldap_pvt_thread_mutex_t si_ops_mutex; ldap_pvt_thread_mutex_t si_mods_mutex; - char si_ctxcsnbuf[LDAP_LUTIL_CSNSTR_BUFSIZE]; } syncprov_info_t; typedef struct opcookie { @@ -590,13 +592,10 @@ syncprov_findcsn( Operation *op, find_csn_t mode ) sync_control *srs = NULL; struct slap_limits_set fc_limits; int i, rc = LDAP_SUCCESS, findcsn_retry = 1; + int maxid = 0; if ( mode != FIND_MAXCSN ) { srs = op->o_controls[slap_cids.sc_LDAPsync]; - - if ( srs->sr_state.ctxcsn.bv_len >= LDAP_LUTIL_CSNSTR_BUFSIZE ) { - return LDAP_OTHER; - } } fop = *op; @@ -606,6 +605,7 @@ syncprov_findcsn( Operation *op, find_csn_t mode ) cf.f_ava = &eq; cf.f_av_desc = slap_schema.si_ad_entryCSN; + BER_BVZERO( &cf.f_av_value ); cf.f_next = NULL; fop.o_callback = &cb; @@ -618,7 +618,14 @@ again: switch( mode ) { case FIND_MAXCSN: cf.f_choice = LDAP_FILTER_GE; - cf.f_av_value = si->si_ctxcsn; + cf.f_av_value = si->si_ctxcsn[0]; + /* If there are multiple CSNs, use the largest */ + for ( i=1; isi_numcsns; i++) { + if ( ber_bvcmp( &cf.f_av_value, &si->si_ctxcsn[i] ) < 0 ) { + cf.f_av_value = si->si_ctxcsn[i]; + maxid = i; + } + } fop.ors_filterstr.bv_len = sprintf( buf, "(entryCSN>=%s)", cf.f_av_value.bv_val ); fop.ors_attrsonly = 0; @@ -626,12 +633,21 @@ again: fop.ors_slimit = SLAP_NO_LIMIT; cb.sc_private = &maxcsn; cb.sc_response = findmax_cb; - strcpy( cbuf, si->si_ctxcsn.bv_val ); + strcpy( cbuf, cf.f_av_value.bv_val ); maxcsn.bv_val = cbuf; - maxcsn.bv_len = si->si_ctxcsn.bv_len; + maxcsn.bv_len = cf.f_av_value.bv_len; break; case FIND_CSN: - cf.f_av_value = srs->sr_state.ctxcsn; + if ( BER_BVISEMPTY( &cf.f_av_value )) { + cf.f_av_value = srs->sr_state.ctxcsn[0]; + /* If there are multiple CSNs, use the smallest */ + for ( i=1; isr_state.numcsns; i++ ) { + if ( ber_bvcmp( &cf.f_av_value, &srs->sr_state.ctxcsn[i] ) + > 0 ) { + cf.f_av_value = srs->sr_state.ctxcsn[i]; + } + } + } /* Look for exact match the first time */ if ( findcsn_retry ) { cf.f_choice = LDAP_FILTER_EQUALITY; @@ -681,8 +697,10 @@ again: switch( mode ) { case FIND_MAXCSN: - strcpy( si->si_ctxcsnbuf, maxcsn.bv_val ); - si->si_ctxcsn.bv_len = maxcsn.bv_len; + if ( ber_bvcmp( &si->si_ctxcsn[maxid], &maxcsn )) { + ber_bvreplace( &si->si_ctxcsn[maxid], &maxcsn ); + si->si_numops++; /* ensure a checkpoint */ + } break; case FIND_CSN: /* If matching CSN was not found, invalidate the context. */ @@ -741,7 +759,7 @@ syncprov_sendresp( Operation *op, opcookie *opc, syncops *so, SlapReply rs = { REP_SEARCH }; LDAPControl *ctrls[2]; - struct berval cookie; + struct berval cookie, csns[2]; Entry e_uuid = {0}; Attribute a_uuid = {0}; @@ -749,7 +767,9 @@ syncprov_sendresp( Operation *op, opcookie *opc, syncops *so, return SLAPD_ABANDON; ctrls[1] = NULL; - slap_compose_sync_cookie( op, &cookie, &opc->sctxcsn, so->s_rid ); + csns[0] = opc->sctxcsn; + BER_BVZERO( &csns[1] ); + slap_compose_sync_cookie( op, &cookie, csns, so->s_rid ); e_uuid.e_attrs = &a_uuid; a_uuid.a_desc = slap_schema.si_ad_entryUUID; @@ -1256,23 +1276,15 @@ syncprov_op_cleanup( Operation *op, SlapReply *rs ) } static void -syncprov_checkpoint( Operation *op, SlapReply *rs, slap_overinst *on, - struct berval *csn ) +syncprov_checkpoint( Operation *op, SlapReply *rs, slap_overinst *on ) { + syncprov_info_t *si = (syncprov_info_t *)on->on_bi.bi_private; Modifications mod; Operation opm; SlapReply rsm = { 0 }; - struct berval bv[2]; slap_callback cb = {0}; - /* If ctxcsn is empty, delete it */ - if ( BER_BVISEMPTY( csn )) { - mod.sml_values = NULL; - } else { - mod.sml_values = bv; - bv[1].bv_val = NULL; - bv[0] = *csn; - } + mod.sml_values = si->si_ctxcsn; mod.sml_nvalues = NULL; mod.sml_desc = slap_schema.si_ad_contextCSN; mod.sml_op = LDAP_MOD_REPLACE; @@ -1284,13 +1296,12 @@ syncprov_checkpoint( Operation *op, SlapReply *rs, slap_overinst *on, opm.o_tag = LDAP_REQ_MODIFY; opm.o_callback = &cb; opm.orm_modlist = &mod; + opm.orm_no_opattrs = 1; opm.o_req_dn = op->o_bd->be_suffix[0]; opm.o_req_ndn = op->o_bd->be_nsuffix[0]; opm.o_bd->bd_info = on->on_info->oi_orig; opm.o_managedsait = SLAP_CONTROL_NONCRITICAL; - SLAP_DBFLAGS( opm.o_bd ) |= SLAP_DBFLAG_NOLASTMOD; opm.o_bd->be_modify( &opm, &rsm ); - SLAP_DBFLAGS( opm.o_bd ) ^= SLAP_DBFLAG_NOLASTMOD; if ( mod.sml_next != NULL ) { slap_mods_free( mod.sml_next, 1 ); } @@ -1321,6 +1332,7 @@ syncprov_add_slog( Operation *op ) AC_MEMCPY( se->se_csn.bv_val, op->o_csn.bv_val, op->o_csn.bv_len ); se->se_csn.bv_val[op->o_csn.bv_len] = '\0'; se->se_csn.bv_len = op->o_csn.bv_len; + se->se_sid = slap_parse_csn_sid( &se->se_csn ); ldap_pvt_thread_mutex_lock( &sl->sl_mutex ); if ( sl->sl_head ) { @@ -1355,14 +1367,14 @@ playlog_cb( Operation *op, SlapReply *rs ) /* enter with sl->sl_mutex locked, release before returning */ static void syncprov_playlog( Operation *op, SlapReply *rs, sessionlog *sl, - sync_control *srs, struct berval *ctxcsn ) + sync_control *srs, BerVarray ctxcsn, int numcsns, int *sids ) { slap_overinst *on = (slap_overinst *)op->o_bd->bd_info; slog_entry *se; int i, j, ndel, num, nmods, mmods; char cbuf[LDAP_LUTIL_CSNSTR_BUFSIZE]; BerVarray uuids; - struct berval delcsn; + struct berval delcsn[2]; if ( !sl->sl_num ) { ldap_pvt_thread_mutex_unlock( &sl->sl_mutex ); @@ -1377,35 +1389,47 @@ syncprov_playlog( Operation *op, SlapReply *rs, sessionlog *sl, num * UUID_LEN, op->o_tmpmemctx ); uuids[0].bv_val = (char *)(uuids + num + 1); - delcsn.bv_len = 0; - delcsn.bv_val = cbuf; + delcsn[0].bv_len = 0; + delcsn[0].bv_val = cbuf; + BER_BVZERO(&delcsn[1]); /* Make a copy of the relevant UUIDs. Put the Deletes up front * and everything else at the end. Do this first so we can * unlock the list mutex. */ - Debug( LDAP_DEBUG_SYNC, "srs csn %s\n", srs-> sr_state.ctxcsn.bv_val, 0, 0 ); + Debug( LDAP_DEBUG_SYNC, "srs csn %s\n", + srs->sr_state.ctxcsn[0].bv_val, 0, 0 ); for ( se=sl->sl_head; se; se=se->se_next ) { - Debug( LDAP_DEBUG_SYNC, "log csn %s\n", se-> se_csn.bv_val, -0, 0 ); - ndel = ber_bvcmp( &se->se_csn, &srs->sr_state.ctxcsn ); - if ( ndel <= 0 ) { - Debug( LDAP_DEBUG_SYNC, "cmp %d, too old\n", ndel, -0, 0 ); - continue; - } - ndel = ber_bvcmp( &se->se_csn, ctxcsn ); - if ( ndel > 0 ) { - Debug( LDAP_DEBUG_SYNC, "cmp %d, too new\n", ndel, -0, 0 ); - break; - } + int k; + Debug( LDAP_DEBUG_SYNC, "log csn %s\n", se->se_csn.bv_val, 0, 0 ); + ndel = 1; + for ( k=0; ksr_state.numcsns; k++ ) { + if ( se->se_sid == srs->sr_state.sids[k] ) { + ndel = ber_bvcmp( &se->se_csn, &srs->sr_state.ctxcsn[k] ); + break; + } + } + if ( ndel <= 0 ) { + Debug( LDAP_DEBUG_SYNC, "cmp %d, too old\n", ndel, 0, 0 ); + continue; + } + ndel = 0; + for ( k=0; kse_sid == sids[k] ) { + ndel = ber_bvcmp( &se->se_csn, &ctxcsn[k] ); + break; + } + } + if ( ndel > 0 ) { + Debug( LDAP_DEBUG_SYNC, "cmp %d, too new\n", ndel, 0, 0 ); + break; + } if ( se->se_tag == LDAP_REQ_DELETE ) { j = i; i++; AC_MEMCPY( cbuf, se->se_csn.bv_val, se->se_csn.bv_len ); - delcsn.bv_len = se->se_csn.bv_len; - delcsn.bv_val[delcsn.bv_len] = '\0'; + delcsn[0].bv_len = se->se_csn.bv_len; + delcsn[0].bv_val[delcsn[0].bv_len] = '\0'; } else { nmods++; j = num - nmods; @@ -1499,7 +1523,7 @@ syncprov_playlog( Operation *op, SlapReply *rs, sessionlog *sl, if ( ndel ) { struct berval cookie; - slap_compose_sync_cookie( op, &cookie, &delcsn, srs->sr_state.rid ); + slap_compose_sync_cookie( op, &cookie, delcsn, srs->sr_state.rid ); uuids[ndel].bv_val = NULL; syncprov_sendinfo( op, rs, LDAP_TAG_SYNC_ID_SET, &cookie, 0, uuids, 1 ); op->o_tmpfree( cookie.bv_val, op->o_tmpmemctx ); @@ -1526,10 +1550,24 @@ syncprov_op_response( Operation *op, SlapReply *rs ) ldap_pvt_thread_mutex_lock( &si->si_csn_mutex ); slap_get_commit_csn( op, &maxcsn ); if ( !BER_BVISNULL( &maxcsn ) ) { + int i, sid; strcpy( cbuf, maxcsn.bv_val ); - if ( ber_bvcmp( &maxcsn, &si->si_ctxcsn ) > 0 ) { - strcpy( si->si_ctxcsnbuf, cbuf ); - si->si_ctxcsn.bv_len = maxcsn.bv_len; + sid = slap_parse_csn_sid( &maxcsn ); + for ( i=0; isi_numcsns; i++ ) { + if ( sid == si->si_sids[i] ) { + if ( ber_bvcmp( &maxcsn, &si->si_ctxcsn[i] ) > 0 ) { + ber_bvreplace( &si->si_ctxcsn[i], &maxcsn ); + } + break; + } + } + /* It's a new SID for us */ + if ( i == si->si_numcsns ) { + value_add_one( &si->si_ctxcsn, &maxcsn ); + si->si_numcsns++; + si->si_sids = ch_realloc( si->si_sids, si->si_numcsns * + sizeof(int)); + si->si_sids[i] = sid; } } @@ -1558,7 +1596,7 @@ syncprov_op_response( Operation *op, SlapReply *rs ) opc->sctxcsn.bv_val = cbuf; if ( do_check ) { - syncprov_checkpoint( op, rs, on, &opc->sctxcsn ); + syncprov_checkpoint( op, rs, on ); } /* Handle any persistent searches */ @@ -1608,20 +1646,18 @@ syncprov_op_compare( Operation *op, SlapReply *rs ) { Entry e = {0}; Attribute a = {0}; - struct berval bv[2]; e.e_name = op->o_bd->be_suffix[0]; e.e_nname = op->o_bd->be_nsuffix[0]; - - BER_BVZERO( &bv[1] ); - bv[0] = si->si_ctxcsn; + e.e_attrs = &a; a.a_desc = slap_schema.si_ad_contextCSN; - a.a_vals = bv; - a.a_nvals = a.a_vals; ldap_pvt_thread_mutex_lock( &si->si_csn_mutex ); + a.a_vals = si->si_ctxcsn; + a.a_nvals = a.a_vals; + rs->sr_err = access_allowed( op, &e, op->oq_compare.rs_ava->aa_desc, &op->oq_compare.rs_ava->aa_value, ACL_COMPARE, NULL ); if ( ! rs->sr_err ) { @@ -1748,9 +1784,10 @@ syncprov_op_extended( Operation *op, SlapReply *rs ) typedef struct searchstate { slap_overinst *ss_on; syncops *ss_so; + BerVarray ss_ctxcsn; + int *ss_sids; + int ss_numcsns; int ss_present; - struct berval ss_ctxcsn; - char ss_csnbuf[LDAP_LUTIL_CSNSTR_BUFSIZE]; } searchstate; static int @@ -1879,23 +1916,36 @@ syncprov_search_response( Operation *op, SlapReply *rs ) a = attr_find( rs->sr_operational_attrs, slap_schema.si_ad_entryCSN ); } if ( a ) { + int i, sid; + sid = slap_parse_csn_sid( &a->a_nvals[0] ); + /* Make sure entry is less than the snapshot'd contextCSN */ - if ( ber_bvcmp( &a->a_nvals[0], &ss->ss_ctxcsn ) > 0 ) { - Debug( LDAP_DEBUG_SYNC, "Entry %s CSN %s greater than snapshot %s\n", - rs->sr_entry->e_name.bv_val, - a->a_nvals[0].bv_val, - ss->ss_ctxcsn.bv_val ); - return LDAP_SUCCESS; + for ( i=0; iss_numcsns; i++ ) { + if ( sid == ss->ss_sids[i] && ber_bvcmp( &a->a_nvals[0], + &ss->ss_ctxcsn[i] ) > 0 ) { + Debug( LDAP_DEBUG_SYNC, + "Entry %s CSN %s greater than snapshot %s\n", + rs->sr_entry->e_name.bv_val, + a->a_nvals[0].bv_val, + ss->ss_ctxcsn[i].bv_val ); + return LDAP_SUCCESS; + } } - /* Don't send the ctx entry twice */ - if ( !BER_BVISNULL( &srs->sr_state.ctxcsn ) && - bvmatch( &a->a_nvals[0], &srs->sr_state.ctxcsn ) ) { - Debug( LDAP_DEBUG_SYNC, "Entry %s CSN %s matches ctx %s\n", - rs->sr_entry->e_name.bv_val, - a->a_nvals[0].bv_val, - srs->sr_state.ctxcsn.bv_val ); - return LDAP_SUCCESS; + /* Don't send old entries twice */ + if ( srs->sr_state.ctxcsn ) { + for ( i=0; isr_state.numcsns; i++ ) { + if ( sid == srs->sr_state.sids[i] && + ber_bvcmp( &a->a_nvals[0], + &srs->sr_state.ctxcsn[i] )<= 0 ) { + Debug( LDAP_DEBUG_SYNC, + "Entry %s CSN %s older or equal to ctx %s\n", + rs->sr_entry->e_name.bv_val, + a->a_nvals[0].bv_val, + srs->sr_state.ctxcsn[i].bv_val ); + return LDAP_SUCCESS; + } + } } } rs->sr_ctrls = op->o_tmpalloc( sizeof(LDAPControl *)*2, @@ -1906,7 +1956,7 @@ syncprov_search_response( Operation *op, SlapReply *rs ) } else if ( rs->sr_type == REP_RESULT && rs->sr_err == LDAP_SUCCESS ) { struct berval cookie; - slap_compose_sync_cookie( op, &cookie, &ss->ss_ctxcsn, + slap_compose_sync_cookie( op, &cookie, ss->ss_ctxcsn, srs->sr_state.rid ); /* Is this a regular refresh? */ @@ -1955,8 +2005,9 @@ syncprov_op_search( Operation *op, SlapReply *rs ) syncops *sop = NULL; searchstate *ss; sync_control *srs; - struct berval ctxcsn; - char csnbuf[LDAP_LUTIL_CSNSTR_BUFSIZE]; + BerVarray ctxcsn; + int i, *sids, numcsns; + struct berval mincsn; if ( !(op->o_sync_mode & SLAP_SYNC_REFRESH) ) return SLAP_CB_CONTINUE; @@ -2010,34 +2061,77 @@ syncprov_op_search( Operation *op, SlapReply *rs ) /* snapshot the ctxcsn */ ldap_pvt_thread_mutex_lock( &si->si_csn_mutex ); - strcpy( csnbuf, si->si_ctxcsnbuf ); - ctxcsn.bv_len = si->si_ctxcsn.bv_len; + ber_bvarray_dup_x( &ctxcsn, si->si_ctxcsn, op->o_tmpmemctx ); + numcsns = si->si_numcsns; + sids = op->o_tmpalloc( numcsns * sizeof(int), op->o_tmpmemctx ); + for ( i=0; isi_sids[i]; ldap_pvt_thread_mutex_unlock( &si->si_csn_mutex ); - ctxcsn.bv_val = csnbuf; /* If we have a cookie, handle the PRESENT lookups */ - if ( !BER_BVISNULL( &srs->sr_state.ctxcsn )) { + if ( srs->sr_state.ctxcsn ) { sessionlog *sl; + int i, j; - /* The cookie was validated when it was parsed, just use it */ + /* If there are SIDs we don't recognize in the cookie, drop them */ + for (i=0; isr_state.numcsns; ) { + for (j=0; jsr_state.sids[i] == sids[j] ) { + break; + } + } + /* not found */ + if ( j == numcsns ) { + struct berval tmp = srs->sr_state.ctxcsn[i]; + j = srs->sr_state.numcsns - 1; + srs->sr_state.ctxcsn[i] = srs->sr_state.ctxcsn[j]; + tmp.bv_len = 0; + srs->sr_state.ctxcsn[j] = tmp; + srs->sr_state.numcsns = j; + srs->sr_state.sids[i] = srs->sr_state.sids[j]; + continue; + } + i++; + } - /* If just Refreshing and nothing has changed, shortcut it */ - if ( bvmatch( &srs->sr_state.ctxcsn, &ctxcsn )) { - nochange = 1; - if ( !(op->o_sync_mode & SLAP_SYNC_PERSIST) ) { - LDAPControl *ctrls[2]; + /* Find the smallest CSN */ + mincsn = srs->sr_state.ctxcsn[0]; + for ( i=1; isr_state.numcsns; i++ ) { + if ( ber_bvcmp( &mincsn, &srs->sr_state.ctxcsn[i] ) > 0 ) + mincsn = srs->sr_state.ctxcsn[i]; + } - ctrls[0] = NULL; - ctrls[1] = NULL; - syncprov_done_ctrl( op, rs, ctrls, 0, 0, - NULL, LDAP_SYNC_REFRESH_DELETES ); - rs->sr_ctrls = ctrls; - rs->sr_err = LDAP_SUCCESS; - send_ldap_result( op, rs ); - rs->sr_ctrls = NULL; - return rs->sr_err; + /* If nothing has changed, shortcut it */ + if ( srs->sr_state.numcsns == numcsns ) { + int i, j, changed = 0; + for ( i=0; isr_state.numcsns; i++ ) { + for ( j=0; jsr_state.sids[i] != sids[j] ) + continue; + if ( !bvmatch( &srs->sr_state.ctxcsn[i], &ctxcsn[j] )) + changed = 1; + break; + } + if ( changed ) + break; + } + if ( !changed ) { + nochange = 1; + if ( !(op->o_sync_mode & SLAP_SYNC_PERSIST) ) { + LDAPControl *ctrls[2]; + + ctrls[0] = NULL; + ctrls[1] = NULL; + syncprov_done_ctrl( op, rs, ctrls, 0, 0, + NULL, LDAP_SYNC_REFRESH_DELETES ); + rs->sr_ctrls = ctrls; + rs->sr_err = LDAP_SUCCESS; + send_ldap_result( op, rs ); + rs->sr_ctrls = NULL; + return rs->sr_err; + } + goto shortcut; } - goto shortcut; } /* Do we have a sessionlog for this search? */ sl=si->si_logs; @@ -2046,10 +2140,10 @@ syncprov_op_search( Operation *op, SlapReply *rs ) /* Are there any log entries, and is the consumer state * present in the session log? */ - if ( sl->sl_num > 0 && ber_bvcmp( &srs->sr_state.ctxcsn, &sl->sl_mincsn ) >= 0 ) { + if ( sl->sl_num > 0 && ber_bvcmp( &mincsn, &sl->sl_mincsn ) >= 0 ) { do_present = 0; /* mutex is unlocked in playlog */ - syncprov_playlog( op, rs, sl, srs, &ctxcsn ); + syncprov_playlog( op, rs, sl, srs, ctxcsn, numcsns, sids ); } else { ldap_pvt_thread_mutex_unlock( &sl->sl_mutex ); } @@ -2096,7 +2190,7 @@ shortcut: #ifdef LDAP_COMP_MATCH fava->f_ava->aa_cf = NULL; #endif - ber_dupbv_x( &fava->f_ava->aa_value, &srs->sr_state.ctxcsn, op->o_tmpmemctx ); + ber_dupbv_x( &fava->f_ava->aa_value, &mincsn, op->o_tmpmemctx ); fava->f_next = op->ors_filter; op->ors_filter = fand; filter2bv_x( op, op->ors_filter, &op->ors_filterstr ); @@ -2110,19 +2204,15 @@ shortcut: ss->ss_on = on; ss->ss_so = sop; ss->ss_present = do_present; - ss->ss_ctxcsn.bv_len = ctxcsn.bv_len; - ss->ss_ctxcsn.bv_val = ss->ss_csnbuf; - strcpy( ss->ss_ctxcsn.bv_val, ctxcsn.bv_val ); + ss->ss_ctxcsn = ctxcsn; + ss->ss_numcsns = numcsns; + ss->ss_sids = sids; cb->sc_response = syncprov_search_response; cb->sc_cleanup = syncprov_search_cleanup; cb->sc_private = ss; cb->sc_next = op->o_callback; op->o_callback = cb; -#if 0 /* I don't think we need to shortcircuit back-bdb any more */ - op->o_sync_mode &= SLAP_CONTROL_MASK; -#endif - /* If this is a persistent search and no changes were reported during * the refresh phase, just invoke the response callback to transition * us into persist phase @@ -2156,21 +2246,28 @@ syncprov_operational( break; } - if ( !a ) { - for ( ap = &rs->sr_operational_attrs; *ap; ap=&(*ap)->a_next ); + ldap_pvt_thread_mutex_lock( &si->si_csn_mutex ); + if ( si->si_ctxcsn ) { + if ( !a ) { + for ( ap = &rs->sr_operational_attrs; *ap; + ap=&(*ap)->a_next ); - a = attr_alloc( slap_schema.si_ad_contextCSN ); - a->a_vals = ch_malloc( 2 * sizeof(struct berval)); - a->a_vals[1].bv_val = NULL; - a->a_nvals = a->a_vals; - *ap = a; - } + a = attr_alloc( slap_schema.si_ad_contextCSN ); + *ap = a; + } - ldap_pvt_thread_mutex_lock( &si->si_csn_mutex ); - if ( !ap ) { - strcpy( a->a_vals[0].bv_val, si->si_ctxcsnbuf ); - } else { - ber_dupbv( &a->a_vals[0], &si->si_ctxcsn ); + if ( !ap ) { + if ( !rs->sr_flags & REP_ENTRY_MODIFIABLE ) { + rs->sr_entry = entry_dup( rs->sr_entry ); + rs->sr_flags |= + REP_ENTRY_MODIFIABLE|REP_ENTRY_MUSTBEFREED; + a = attr_find( rs->sr_entry->e_attrs, + slap_schema.si_ad_contextCSN ); + } + free( a->a_vals ); + } + ber_bvarray_dup_x( &a->a_vals, si->si_ctxcsn, NULL ); + a->a_nvals = a->a_vals; } ldap_pvt_thread_mutex_unlock( &si->si_csn_mutex ); } @@ -2379,7 +2476,6 @@ syncprov_db_open( Connection conn = { 0 }; OperationBuffer opbuf = { 0 }; - char ctxcsnbuf[LDAP_LUTIL_CSNSTR_BUFSIZE]; Operation *op = (Operation *) &opbuf; Entry *e; Attribute *a; @@ -2407,8 +2503,6 @@ syncprov_db_open( op->o_dn = be->be_rootdn; op->o_ndn = be->be_rootndn; - ctxcsnbuf[0] = '\0'; - op->o_bd->bd_info = on->on_info->oi_orig; rc = be_entry_get_rw( op, be->be_nsuffix, NULL, slap_schema.si_ad_contextCSN, 0, &e ); @@ -2418,16 +2512,14 @@ syncprov_db_open( a = attr_find( e->e_attrs, slap_schema.si_ad_contextCSN ); if ( a ) { - si->si_ctxcsn.bv_len = a->a_nvals[0].bv_len; - if ( si->si_ctxcsn.bv_len >= sizeof(si->si_ctxcsnbuf )) - si->si_ctxcsn.bv_len = sizeof(si->si_ctxcsnbuf)-1; - strncpy( si->si_ctxcsnbuf, a->a_nvals[0].bv_val, - si->si_ctxcsn.bv_len ); - si->si_ctxcsnbuf[si->si_ctxcsn.bv_len] = '\0'; - strcpy( ctxcsnbuf, si->si_ctxcsnbuf ); + int i; + ber_bvarray_dup_x( &si->si_ctxcsn, a->a_vals, NULL ); + for ( i=0; !BER_BVISEMPTY( &a->a_vals[i] ); i++ ); + si->si_numcsns = i; + si->si_sids = slap_parse_csn_sids( si->si_ctxcsn, i ); } be_entry_release_rw( op, e, 0 ); - if ( !BER_BVISEMPTY( &si->si_ctxcsn ) ) { + if ( si->si_ctxcsn ) { op->o_bd->bd_info = (BackendInfo *)on; op->o_req_dn = be->be_suffix[0]; op->o_req_ndn = be->be_nsuffix[0]; @@ -2437,22 +2529,27 @@ syncprov_db_open( } } - if ( BER_BVISEMPTY( &si->si_ctxcsn ) ) { - if ( SLAP_SYNC_SHADOW( op->o_bd )) { - /* If we're also a consumer, and we didn't get a contextCSN, + /* Didn't find a contextCSN, should we generate one? */ + if ( !si->si_ctxcsn ) { + char csnbuf[ LDAP_LUTIL_CSNSTR_BUFSIZE ]; + struct berval csn; + + if ( SLAP_SYNC_SHADOW( op->o_bd ) && SLAP_SINGLE_SHADOW( op->o_bd )) { + /* If we're also a consumer, and we're not multimaster, * then don't generate anything, wait for our provider to send it * to us. */ goto out; } - si->si_ctxcsn.bv_len = sizeof( si->si_ctxcsnbuf ); - slap_get_csn( op, &si->si_ctxcsn, 0 ); - } + csn.bv_val = csnbuf; + csn.bv_len = sizeof( csnbuf ); + slap_get_csn( op, &csn, 0 ); + value_add_one( &si->si_ctxcsn, &csn ); + si->si_numcsns = 1; + si->si_sids = ch_malloc( sizeof(int) ); + si->si_sids[0] = slap_serverID; - /* If our ctxcsn is different from what was read from the root - * entry, make sure we do a checkpoint on close - */ - if ( strcmp( si->si_ctxcsnbuf, ctxcsnbuf )) { + /* make sure we do a checkpoint on close */ si->si_numops++; } @@ -2486,7 +2583,7 @@ syncprov_db_close( op->o_bd = be; op->o_dn = be->be_rootdn; op->o_ndn = be->be_rootndn; - syncprov_checkpoint( op, &rs, on, &si->si_ctxcsn ); + syncprov_checkpoint( op, &rs, on ); } return 0; @@ -2512,7 +2609,6 @@ syncprov_db_init( ldap_pvt_thread_mutex_init( &si->si_csn_mutex ); ldap_pvt_thread_mutex_init( &si->si_ops_mutex ); ldap_pvt_thread_mutex_init( &si->si_mods_mutex ); - si->si_ctxcsn.bv_val = si->si_ctxcsnbuf; csn_anlist[0].an_desc = slap_schema.si_ad_entryCSN; csn_anlist[0].an_name = slap_schema.si_ad_entryCSN->ad_cname; diff --git a/servers/slapd/proto-slap.h b/servers/slapd/proto-slap.h index 772500add1..d04794faf3 100644 --- a/servers/slapd/proto-slap.h +++ b/servers/slapd/proto-slap.h @@ -1016,9 +1016,13 @@ LDAP_SLAPD_V (char *) slap_known_controls[]; * ldapsync.c */ LDAP_SLAPD_F (void) slap_compose_sync_cookie LDAP_P(( - Operation *, struct berval *, struct berval *, int )); + Operation *, struct berval *, BerVarray, int )); LDAP_SLAPD_F (void) slap_sync_cookie_free LDAP_P(( struct sync_cookie *, int free_cookie )); +LDAP_SLAPD_F (int) slap_parse_csn_sid LDAP_P(( + struct berval * )); +LDAP_SLAPD_F (int *) slap_parse_csn_sids LDAP_P(( + BerVarray, int )); LDAP_SLAPD_F (int) slap_parse_sync_cookie LDAP_P(( struct sync_cookie *, void *memctx )); LDAP_SLAPD_F (int) slap_init_sync_cookie_ctxcsn LDAP_P(( diff --git a/servers/slapd/schema_prep.c b/servers/slapd/schema_prep.c index cad035f1bf..43eb1be766 100644 --- a/servers/slapd/schema_prep.c +++ b/servers/slapd/schema_prep.c @@ -603,7 +603,7 @@ static struct slap_schema_ad_map { "EQUALITY CSNMatch " "ORDERING CSNOrderingMatch " "SYNTAX 1.3.6.1.4.1.4203.666.11.2.1{64} " - "SINGLE-VALUE NO-USER-MODIFICATION USAGE dSAOperation )", + "NO-USER-MODIFICATION USAGE dSAOperation )", NULL, SLAP_AT_HIDE, NULL, NULL, NULL, NULL, NULL, NULL, NULL, diff --git a/servers/slapd/slap.h b/servers/slapd/slap.h index 3acd543545..720046d25c 100644 --- a/servers/slapd/slap.h +++ b/servers/slapd/slap.h @@ -1650,14 +1650,17 @@ typedef BackendDB Backend; struct syncinfo_s; #define SLAP_SYNC_RID_SIZE 3 +#define SLAP_SYNC_SID_MAX 4095 /* based on liblutil/csn.c field width */ #define SLAP_SYNCUUID_SET_SIZE 256 #define SLAP_SYNC_UPDATE_MSGID 1 struct sync_cookie { - struct berval ctxcsn; + struct berval *ctxcsn; struct berval octet_str; - long rid; + int rid; + int numcsns; + int *sids; LDAP_STAILQ_ENTRY(sync_cookie) sc_next; }; diff --git a/servers/slapd/syncrepl.c b/servers/slapd/syncrepl.c index 75fcc6662a..41c45f9532 100644 --- a/servers/slapd/syncrepl.c +++ b/servers/slapd/syncrepl.c @@ -37,6 +37,14 @@ struct nonpresent_entry { LDAP_LIST_ENTRY(nonpresent_entry) npe_link; }; +typedef struct cookie_state { + ldap_pvt_thread_mutex_t cs_mutex; + int cs_num; + int cs_age; + struct berval *cs_vals; + int *cs_sids; +} cookie_state; + #define SYNCDATA_DEFAULT 0 /* entries are plain LDAP entries */ #define SYNCDATA_ACCESSLOG 1 /* entries are accesslog format */ #define SYNCDATA_CHANGELOG 2 /* entries are changelog format */ @@ -50,9 +58,11 @@ struct nonpresent_entry { #define RETRYNUM_FINITE(n) ((n) > RETRYNUM_FOREVER) /* not forever */ typedef struct syncinfo_s { + struct syncinfo_s *si_next; struct slap_backend_db *si_be; struct re_s *si_re; - long si_rid; + int si_rid; + char si_ridtxt[8]; slap_bindconf si_bindconf; struct berval si_base; struct berval si_logbase; @@ -75,6 +85,8 @@ typedef struct syncinfo_s { int *si_retrynum_init; int *si_retrynum; struct sync_cookie si_syncCookie; + cookie_state *si_cookieState; + int si_cookieAge; int si_manageDSAit; int si_slimit; int si_tlimit; @@ -100,8 +112,7 @@ static int syncrepl_message_to_entry( static int syncrepl_entry( syncinfo_t *, Operation*, Entry*, Modifications**,int, struct berval*, - struct sync_cookie *, - struct berval * ); + struct sync_cookie * ); static int syncrepl_updateCookie( syncinfo_t *, Operation *, struct berval *, struct sync_cookie * ); @@ -330,7 +341,7 @@ ldap_sync_search( /* If we're using a log but we have no state, then fallback to * normal mode for a full refresh. */ - if ( si->si_syncdata && BER_BVISEMPTY( &si->si_syncCookie.ctxcsn ) ) { + if ( si->si_syncdata && !si->si_syncCookie.numcsns ) { si->si_logstate = SYNCLOG_FALLBACK; } @@ -417,13 +428,10 @@ do_syncrep1( int cmdline_cookie_found = 0; struct sync_cookie *sc = NULL; - struct berval *psub; #ifdef HAVE_TLS void *ssl; #endif - psub = &si->si_be->be_nsuffix[0]; - rc = slap_client_connect( &si->si_ld, &si->si_bindconf ); if ( rc != LDAP_SUCCESS ) { goto done; @@ -447,21 +455,43 @@ do_syncrep1( if ( BER_BVISNULL( &si->si_syncCookie.octet_str ) ) { - /* get contextCSN shadow replica from database */ - BerVarray csn = NULL; - - assert( si->si_rid < 1000 ); - op->o_req_ndn = op->o_bd->be_nsuffix[0]; - op->o_req_dn = op->o_req_ndn; + int i; - /* try to read stored contextCSN */ - backend_attribute( op, NULL, &op->o_req_ndn, - slap_schema.si_ad_contextCSN, &csn, ACL_READ ); - if ( csn ) { - ch_free( si->si_syncCookie.ctxcsn.bv_val ); - ber_dupbv( &si->si_syncCookie.ctxcsn, csn ); - ber_bvarray_free_x( csn, op->o_tmpmemctx ); + ldap_pvt_thread_mutex_lock( &si->si_cookieState->cs_mutex ); + if ( !si->si_cookieState->cs_num ) { + /* get contextCSN shadow replica from database */ + BerVarray csn = NULL; + void *ctx = op->o_tmpmemctx; + + op->o_req_ndn = op->o_bd->be_nsuffix[0]; + op->o_req_dn = op->o_req_ndn; + + /* try to read stored contextCSN */ + op->o_tmpmemctx = NULL; + backend_attribute( op, NULL, &op->o_req_ndn, + slap_schema.si_ad_contextCSN, &csn, ACL_READ ); + op->o_tmpmemctx = ctx; + if ( csn ) { + si->si_cookieState->cs_vals = csn; + for (i=0; !BER_BVISNULL( &csn[i] ); i++); + si->si_cookieState->cs_num = i; + si->si_cookieState->cs_sids = slap_parse_csn_sids( csn, i ); + } + } + if ( si->si_cookieState->cs_num ) { + ber_bvarray_free( si->si_syncCookie.ctxcsn ); + if ( ber_bvarray_dup_x( &si->si_syncCookie.ctxcsn, + si->si_cookieState->cs_vals, NULL )) { + rc = LDAP_NO_MEMORY; + goto done; + } + si->si_syncCookie.numcsns = si->si_cookieState->cs_num; + si->si_syncCookie.sids = ch_malloc( si->si_cookieState->cs_num * + sizeof(int) ); + for ( i=0; isi_syncCookie.numcsns; i++ ) + si->si_syncCookie.sids[i] = si->si_cookieState->cs_sids[i]; } + ldap_pvt_thread_mutex_unlock( &si->si_cookieState->cs_mutex ); si->si_syncCookie.rid = si->si_rid; @@ -479,7 +509,7 @@ do_syncrep1( /* ctxcsn wasn't parsed yet, do it now */ slap_parse_sync_cookie( sc, op->o_tmpmemctx ); - if ( BER_BVISNULL( &sc->ctxcsn ) ) { + if ( !sc->ctxcsn ) { /* if cmdline cookie does not have ctxcsn */ /* component, set it to an initial value */ slap_init_sync_cookie_ctxcsn( sc ); @@ -490,15 +520,41 @@ do_syncrep1( } slap_compose_sync_cookie( NULL, &si->si_syncCookie.octet_str, - &si->si_syncCookie.ctxcsn, si->si_syncCookie.rid ); + si->si_syncCookie.ctxcsn, si->si_syncCookie.rid ); + } else { + ldap_pvt_thread_mutex_lock( &si->si_cookieState->cs_mutex ); + /* match SIDs */ + if ( si->si_cookieState->cs_num > 1 && si->si_cookieAge != + si->si_cookieState->cs_age ) { + int i, j; + + for (i=0; !BER_BVISNULL( &si->si_syncCookie.ctxcsn[i] ); i++) { + /* bogus, just dup everything */ + if ( si->si_syncCookie.sids[i] == -1 ) { + ber_bvarray_free( si->si_syncCookie.ctxcsn ); + ber_bvarray_dup_x( &si->si_syncCookie.ctxcsn, + si->si_cookieState->cs_vals, NULL ); + break; + } + for (j=0; jsi_cookieState->cs_num; j++) { + if ( si->si_syncCookie.sids[i] != + si->si_cookieState->cs_sids[j] ) + continue; + ber_bvreplace( &si->si_syncCookie.ctxcsn[i], + &si->si_cookieState->cs_vals[j] ); + break; + } + } + } + ldap_pvt_thread_mutex_unlock( &si->si_cookieState->cs_mutex ); } rc = ldap_sync_search( si, op->o_tmpmemctx ); if( rc != LDAP_SUCCESS ) { - Debug( LDAP_DEBUG_ANY, "do_syncrep1: rid %03ld " + Debug( LDAP_DEBUG_ANY, "do_syncrep1: %s " "ldap_search_ext: %s (%d)\n", - si->si_rid, ldap_err2string( rc ), rc ); + si->si_ridtxt, ldap_err2string( rc ), rc ); } done: @@ -512,6 +568,32 @@ done: return rc; } +static int +compare_csns( struct sync_cookie *sc1, struct sync_cookie *sc2, int *which ) +{ + int i, j, match = 0; + const char *text; + + *which = 0; + + for (i=0; !BER_BVISNULL( &sc1->ctxcsn[i] ); i++) { + for (j=0; !BER_BVISNULL( &sc2->ctxcsn[j] ); j++) { + if ( sc1->sids[i] != sc2->sids[j] ) + continue; + value_match( &match, slap_schema.si_ad_entryCSN, + slap_schema.si_ad_entryCSN->ad_type->sat_ordering, + SLAP_MR_VALUE_OF_ATTRIBUTE_SYNTAX, + &sc1->ctxcsn[i], &sc2->ctxcsn[i], &text ); + if ( match < 0 ) { + *which = j; + return match; + } + break; + } + } + return match; +} + static int do_syncrep2( Operation *op, @@ -533,8 +615,8 @@ do_syncrep2( int syncstate; struct berval syncUUID = BER_BVNULL; - struct sync_cookie syncCookie = { BER_BVNULL }; - struct sync_cookie syncCookie_req = { BER_BVNULL }; + struct sync_cookie syncCookie = { NULL }; + struct sync_cookie syncCookie_req = { NULL }; struct berval cookie = BER_BVNULL; int rc, err; @@ -543,8 +625,7 @@ do_syncrep2( struct berval *psub; Modifications *modlist = NULL; - const char *text; - int match; + int match, m; struct timeval *tout_p = NULL; struct timeval tout = { 0, 0 }; @@ -562,7 +643,7 @@ do_syncrep2( ber_init2( ber, NULL, LBER_USE_DER ); ber_set_option( ber, LBER_OPT_BER_MEMCTX, &op->o_tmpmemctx ); - Debug( LDAP_DEBUG_TRACE, "=>do_syncrep2 rid %03ld\n", si->si_rid, 0, 0 ); + Debug( LDAP_DEBUG_TRACE, "=>do_syncrep2 %s\n", si->si_ridtxt, 0, 0 ); psub = &si->si_be->be_nsuffix[0]; @@ -601,9 +682,9 @@ do_syncrep2( rctrlp = ldap_find_control( LDAP_CONTROL_SYNC_STATE, rctrls ); } if ( rctrlp == NULL ) { - Debug( LDAP_DEBUG_ANY, "do_syncrep2: rid %03ld " + Debug( LDAP_DEBUG_ANY, "do_syncrep2: %s " "got search entry without " - "Sync State control\n", si->si_rid, 0, 0 ); + "Sync State control\n", si->si_ridtxt, 0, 0 ); rc = -1; goto done; } @@ -612,8 +693,8 @@ do_syncrep2( /* FIXME: what if syncUUID is NULL or empty? * (happens with back-sql...) */ if ( BER_BVISEMPTY( &syncUUID ) ) { - Debug( LDAP_DEBUG_ANY, "do_syncrep2: rid %03ld " - "got empty syncUUID\n", si->si_rid, 0, 0 ); + Debug( LDAP_DEBUG_ANY, "do_syncrep2: %s " + "got empty syncUUID\n", si->si_ridtxt, 0, 0 ); ldap_controls_free( rctrls ); rc = -1; goto done; @@ -633,7 +714,7 @@ do_syncrep2( if ( si->si_syncdata && si->si_logstate == SYNCLOG_LOGGING ) { modlist = NULL; if ( ( rc = syncrepl_message_to_op( si, op, msg ) ) == LDAP_SUCCESS && - !BER_BVISNULL( &syncCookie.ctxcsn ) ) + syncCookie.ctxcsn ) { rc = syncrepl_updateCookie( si, op, psub, &syncCookie ); } @@ -641,9 +722,8 @@ do_syncrep2( &modlist, &entry, syncstate ) ) == LDAP_SUCCESS ) { if ( ( rc = syncrepl_entry( si, op, entry, &modlist, - syncstate, &syncUUID, &syncCookie_req, - &syncCookie.ctxcsn ) ) == LDAP_SUCCESS && - !BER_BVISNULL( &syncCookie.ctxcsn ) ) + syncstate, &syncUUID, &syncCookie_req ) ) == LDAP_SUCCESS && + syncCookie.ctxcsn ) { rc = syncrepl_updateCookie( si, op, psub, &syncCookie ); } @@ -658,14 +738,14 @@ do_syncrep2( case LDAP_RES_SEARCH_REFERENCE: Debug( LDAP_DEBUG_ANY, - "do_syncrep2: rid %03ld reference received error\n", - si->si_rid, 0, 0 ); + "do_syncrep2: %s reference received error\n", + si->si_ridtxt, 0, 0 ); break; case LDAP_RES_SEARCH_RESULT: Debug( LDAP_DEBUG_SYNC, - "do_syncrep2: rid %03ld LDAP_RES_SEARCH_RESULT\n", - si->si_rid, 0, 0 ); + "do_syncrep2: %s LDAP_RES_SEARCH_RESULT\n", + si->si_ridtxt, 0, 0 ); ldap_parse_result( si->si_ld, msg, &err, NULL, NULL, NULL, &rctrls, 0 ); #ifdef LDAP_X_SYNC_REFRESH_REQUIRED @@ -703,16 +783,12 @@ do_syncrep2( } ber_scanf( ber, /*"{"*/ "}" ); } - if ( BER_BVISNULL( &syncCookie_req.ctxcsn ) ) { + if ( !syncCookie_req.ctxcsn ) { match = -1; - } else if ( BER_BVISNULL( &syncCookie.ctxcsn ) ) { + } else if ( !syncCookie.ctxcsn ) { match = 1; } else { - value_match( &match, slap_schema.si_ad_entryCSN, - slap_schema.si_ad_entryCSN->ad_type->sat_ordering, - SLAP_MR_VALUE_OF_ATTRIBUTE_SYNTAX, - &syncCookie_req.ctxcsn, &syncCookie.ctxcsn, - &text ); + match = compare_csns( &syncCookie_req, &syncCookie, &m ); } if ( rctrls ) { ldap_controls_free( rctrls ); @@ -725,14 +801,14 @@ do_syncrep2( if ( refreshDeletes == 0 && match < 0 && err == LDAP_SUCCESS ) { - syncrepl_del_nonpresent( op, si, NULL, &syncCookie.ctxcsn ); + syncrepl_del_nonpresent( op, si, NULL, + &syncCookie.ctxcsn[m] ); } else { avl_free( si->si_presentlist, avl_ber_bvfree ); si->si_presentlist = NULL; } } - if ( !BER_BVISNULL( &syncCookie.ctxcsn ) && - match < 0 && err == LDAP_SUCCESS ) + if ( syncCookie.ctxcsn && match < 0 && err == LDAP_SUCCESS ) { rc = syncrepl_updateCookie( si, op, psub, &syncCookie ); } @@ -756,8 +832,8 @@ do_syncrep2( ber_tag_t tag; case LDAP_TAG_SYNC_NEW_COOKIE: Debug( LDAP_DEBUG_SYNC, - "do_syncrep2: rid %03ld %s - %s\n", - si->si_rid, + "do_syncrep2: %s %s - %s\n", + si->si_ridtxt, "LDAP_RES_INTERMEDIATE", "NEW_COOKIE" ); ber_scanf( ber, "tm", &tag, &cookie ); @@ -765,8 +841,8 @@ do_syncrep2( case LDAP_TAG_SYNC_REFRESH_DELETE: case LDAP_TAG_SYNC_REFRESH_PRESENT: Debug( LDAP_DEBUG_SYNC, - "do_syncrep2: rid %03ld %s - %s\n", - si->si_rid, + "do_syncrep2: %s %s - %s\n", + si->si_ridtxt, "LDAP_RES_INTERMEDIATE", si_tag == LDAP_TAG_SYNC_REFRESH_PRESENT ? "REFRESH_PRESENT" : "REFRESH_DELETE" ); @@ -797,8 +873,8 @@ do_syncrep2( break; case LDAP_TAG_SYNC_ID_SET: Debug( LDAP_DEBUG_SYNC, - "do_syncrep2: rid %03ld %s - %s\n", - si->si_rid, + "do_syncrep2: %s %s - %s\n", + si->si_ridtxt, "LDAP_RES_INTERMEDIATE", "SYNC_ID_SET" ); ber_scanf( ber, "t{" /*"}"*/, &tag ); @@ -813,6 +889,7 @@ do_syncrep2( if ( !BER_BVISNULL( &syncCookie.octet_str ) ) { slap_parse_sync_cookie( &syncCookie, NULL ); + compare_csns( &syncCookie_req, &syncCookie, &m ); } } if ( ber_peek_tag( ber, &len ) == @@ -824,7 +901,7 @@ do_syncrep2( ber_scanf( ber, /*"{"*/ "}" ); if ( refreshDeletes ) { syncrepl_del_nonpresent( op, si, syncUUIDs, - &syncCookie.ctxcsn ); + &syncCookie.ctxcsn[m] ); ber_bvarray_free_x( syncUUIDs, op->o_tmpmemctx ); } else { int i; @@ -845,31 +922,28 @@ do_syncrep2( break; default: Debug( LDAP_DEBUG_ANY, - "do_syncrep2: rid %03ld unknown syncinfo tag (%ld)\n", - si->si_rid, (long) si_tag, 0 ); + "do_syncrep2: %s unknown syncinfo tag (%ld)\n", + si->si_ridtxt, (long) si_tag, 0 ); ldap_memfree( retoid ); ber_bvfree( retdata ); continue; } - if ( BER_BVISNULL( &syncCookie_req.ctxcsn ) ) { + if ( !syncCookie_req.ctxcsn ) { match = -1; - } else if ( BER_BVISNULL( &syncCookie.ctxcsn ) ) { + } else if ( !syncCookie.ctxcsn ) { match = 1; } else { - value_match( &match, slap_schema.si_ad_entryCSN, - slap_schema.si_ad_entryCSN->ad_type->sat_ordering, - SLAP_MR_VALUE_OF_ATTRIBUTE_SYNTAX, - &syncCookie_req.ctxcsn, - &syncCookie.ctxcsn, &text ); + match = compare_csns( &syncCookie_req, &syncCookie, &m ); } if ( match < 0 ) { if ( si->si_refreshPresent == 1 ) { - syncrepl_del_nonpresent( op, si, NULL, &syncCookie.ctxcsn ); + syncrepl_del_nonpresent( op, si, NULL, + &syncCookie.ctxcsn[m] ); } - if ( !BER_BVISNULL( &syncCookie.ctxcsn ) ) + if ( syncCookie.ctxcsn ) { rc = syncrepl_updateCookie( si, op, psub, &syncCookie); } @@ -880,9 +954,9 @@ do_syncrep2( break; } else { - Debug( LDAP_DEBUG_ANY, "do_syncrep2: rid %03ld " + Debug( LDAP_DEBUG_ANY, "do_syncrep2: %s " "unknown intermediate response (%d)\n", - si->si_rid, rc, 0 ); + si->si_ridtxt, rc, 0 ); ldap_memfree( retoid ); ber_bvfree( retdata ); break; @@ -890,8 +964,8 @@ do_syncrep2( break; default: - Debug( LDAP_DEBUG_ANY, "do_syncrep2: rid %03ld " - "unknown message\n", si->si_rid, 0, 0 ); + Debug( LDAP_DEBUG_ANY, "do_syncrep2: %s " + "unknown message\n", si->si_ridtxt, 0, 0 ); break; } @@ -912,7 +986,7 @@ do_syncrep2( errstr = ldap_err2string( rc ); Debug( LDAP_DEBUG_ANY, - "do_syncrep2: rid %03ld %s\n", si->si_rid, errstr, 0 ); + "do_syncrep2: %s %s\n", si->si_ridtxt, errstr, 0 ); } done: @@ -951,7 +1025,7 @@ do_syncrepl( int i, defer = 1; Backend *be; - Debug( LDAP_DEBUG_TRACE, "=>do_syncrepl rid %03ld\n", si->si_rid, 0, 0 ); + Debug( LDAP_DEBUG_TRACE, "=>do_syncrepl %s\n", si->si_ridtxt, 0, 0 ); if ( si == NULL ) return NULL; @@ -1201,9 +1275,9 @@ syncrepl_message_to_op( int rc, deleteOldRdn = 0, freeReqDn = 0; if ( ldap_msgtype( msg ) != LDAP_RES_SEARCH_ENTRY ) { - Debug( LDAP_DEBUG_ANY, "syncrepl_message_to_op: rid %03ld " + Debug( LDAP_DEBUG_ANY, "syncrepl_message_to_op: %s " "Message type should be entry (%d)", - si->si_rid, ldap_msgtype( msg ), 0 ); + si->si_ridtxt, ldap_msgtype( msg ), 0 ); return -1; } @@ -1216,8 +1290,8 @@ syncrepl_message_to_op( if ( rc != LDAP_SUCCESS ) { Debug( LDAP_DEBUG_ANY, - "syncrepl_message_to_op: rid %03ld dn get failed (%d)", - si->si_rid, rc, 0 ); + "syncrepl_message_to_op: %s dn get failed (%d)", + si->si_ridtxt, rc, 0 ); return rc; } @@ -1240,8 +1314,8 @@ syncrepl_message_to_op( int i = verb_to_mask( bvals[0].bv_val, modops ); if ( i < 0 ) { Debug( LDAP_DEBUG_ANY, - "syncrepl_message_to_op: rid %03ld unknown op %s", - si->si_rid, bvals[0].bv_val, 0 ); + "syncrepl_message_to_op: %s unknown op %s", + si->si_ridtxt, bvals[0].bv_val, 0 ); ch_free( bvals ); rc = -1; goto done; @@ -1288,9 +1362,9 @@ syncrepl_message_to_op( rc = slap_mods_check( op, modlist, &text, txtbuf, textlen, NULL ); if ( rc != LDAP_SUCCESS ) { - Debug( LDAP_DEBUG_ANY, "syncrepl_message_to_op: rid %03ld " + Debug( LDAP_DEBUG_ANY, "syncrepl_message_to_op: %s " "mods check (%s)\n", - si->si_rid, text, 0 ); + si->si_ridtxt, text, 0 ); goto done; } @@ -1302,14 +1376,14 @@ syncrepl_message_to_op( freeReqDn = 0; rc = slap_mods2entry( modlist, &op->ora_e, 1, 0, &text, txtbuf, textlen); if( rc != LDAP_SUCCESS ) { - Debug( LDAP_DEBUG_ANY, "syncrepl_message_to_op: rid %03ld " + Debug( LDAP_DEBUG_ANY, "syncrepl_message_to_op: %s " "mods2entry (%s)\n", - si->si_rid, text, 0 ); + si->si_ridtxt, text, 0 ); } else { rc = op->o_bd->be_add( op, &rs ); Debug( LDAP_DEBUG_SYNC, - "syncrepl_message_to_op: rid %03ld be_add %s (%d)\n", - si->si_rid, op->o_req_dn.bv_val, rc ); + "syncrepl_message_to_op: %s be_add %s (%d)\n", + si->si_ridtxt, op->o_req_dn.bv_val, rc ); } if ( e == op->ora_e ) be_entry_release_w( op, op->ora_e ); @@ -1317,8 +1391,8 @@ syncrepl_message_to_op( op->orm_modlist = modlist; rc = op->o_bd->be_modify( op, &rs ); Debug( rc ? LDAP_DEBUG_ANY : LDAP_DEBUG_SYNC, - "syncrepl_message_to_op: rid %03ld be_modify %s (%d)\n", - si->si_rid, op->o_req_dn.bv_val, rc ); + "syncrepl_message_to_op: %s be_modify %s (%d)\n", + si->si_ridtxt, op->o_req_dn.bv_val, rc ); } break; case LDAP_REQ_MODRDN: @@ -1360,14 +1434,14 @@ syncrepl_message_to_op( rc = op->o_bd->be_modrdn( op, &rs ); slap_mods_free( op->orr_modlist, 1 ); Debug( rc ? LDAP_DEBUG_ANY : LDAP_DEBUG_SYNC, - "syncrepl_message_to_op: rid %03ld be_modrdn %s (%d)\n", - si->si_rid, op->o_req_dn.bv_val, rc ); + "syncrepl_message_to_op: %s be_modrdn %s (%d)\n", + si->si_ridtxt, op->o_req_dn.bv_val, rc ); break; case LDAP_REQ_DELETE: rc = op->o_bd->be_delete( op, &rs ); Debug( rc ? LDAP_DEBUG_ANY : LDAP_DEBUG_SYNC, - "syncrepl_message_to_op: rid %03ld be_delete %s (%d)\n", - si->si_rid, op->o_req_dn.bv_val, rc ); + "syncrepl_message_to_op: %s be_delete %s (%d)\n", + si->si_ridtxt, op->o_req_dn.bv_val, rc ); break; } done: @@ -1425,9 +1499,9 @@ syncrepl_message_to_entry( *modlist = NULL; if ( ldap_msgtype( msg ) != LDAP_RES_SEARCH_ENTRY ) { - Debug( LDAP_DEBUG_ANY, "syncrepl_message_to_entry: rid %03ld " + Debug( LDAP_DEBUG_ANY, "syncrepl_message_to_entry: %s " "Message type should be entry (%d)", - si->si_rid, ldap_msgtype( msg ), 0 ); + si->si_ridtxt, ldap_msgtype( msg ), 0 ); return -1; } @@ -1436,8 +1510,8 @@ syncrepl_message_to_entry( rc = ldap_get_dn_ber( si->si_ld, msg, &ber, &bdn ); if ( rc != LDAP_SUCCESS ) { Debug( LDAP_DEBUG_ANY, - "syncrepl_message_to_entry: rid %03ld dn get failed (%d)", - si->si_rid, rc, 0 ); + "syncrepl_message_to_entry: %s dn get failed (%d)", + si->si_ridtxt, rc, 0 ); return rc; } @@ -1485,8 +1559,8 @@ syncrepl_message_to_entry( } if ( *modlist == NULL ) { - Debug( LDAP_DEBUG_ANY, "syncrepl_message_to_entry: rid %03ld no attributes\n", - si->si_rid, 0, 0 ); + Debug( LDAP_DEBUG_ANY, "syncrepl_message_to_entry: %s no attributes\n", + si->si_ridtxt, 0, 0 ); rc = -1; goto done; } @@ -1494,8 +1568,8 @@ syncrepl_message_to_entry( rc = slap_mods_check( op, *modlist, &text, txtbuf, textlen, NULL ); if ( rc != LDAP_SUCCESS ) { - Debug( LDAP_DEBUG_ANY, "syncrepl_message_to_entry: rid %03ld mods check (%s)\n", - si->si_rid, text, 0 ); + Debug( LDAP_DEBUG_ANY, "syncrepl_message_to_entry: %s mods check (%s)\n", + si->si_ridtxt, text, 0 ); goto done; } @@ -1527,8 +1601,8 @@ syncrepl_message_to_entry( rc = slap_mods2entry( *modlist, &e, 1, 1, &text, txtbuf, textlen); if( rc != LDAP_SUCCESS ) { - Debug( LDAP_DEBUG_ANY, "syncrepl_message_to_entry: rid %03ld mods2entry (%s)\n", - si->si_rid, text, 0 ); + Debug( LDAP_DEBUG_ANY, "syncrepl_message_to_entry: %s mods2entry (%s)\n", + si->si_ridtxt, text, 0 ); } done: @@ -1583,8 +1657,7 @@ syncrepl_entry( Modifications** modlist, int syncstate, struct berval* syncUUID, - struct sync_cookie* syncCookie_req, - struct berval* syncCSN ) + struct sync_cookie* syncCookie_req ) { Backend *be = op->o_bd; slap_callback cb = { NULL, NULL, NULL, NULL }; @@ -1610,28 +1683,28 @@ syncrepl_entry( switch( syncstate ) { case LDAP_SYNC_PRESENT: - Debug( LDAP_DEBUG_SYNC, "syncrepl_entry: rid %03ld %s\n", - si->si_rid, + Debug( LDAP_DEBUG_SYNC, "syncrepl_entry: %s %s\n", + si->si_ridtxt, "LDAP_RES_SEARCH_ENTRY(LDAP_SYNC_PRESENT)", 0 ); break; case LDAP_SYNC_ADD: - Debug( LDAP_DEBUG_SYNC, "syncrepl_entry: rid %03ld %s\n", - si->si_rid, + Debug( LDAP_DEBUG_SYNC, "syncrepl_entry: %s %s\n", + si->si_ridtxt, "LDAP_RES_SEARCH_ENTRY(LDAP_SYNC_ADD)", 0 ); break; case LDAP_SYNC_DELETE: - Debug( LDAP_DEBUG_SYNC, "syncrepl_entry: rid %03ld %s\n", - si->si_rid, + Debug( LDAP_DEBUG_SYNC, "syncrepl_entry: %s %s\n", + si->si_ridtxt, "LDAP_RES_SEARCH_ENTRY(LDAP_SYNC_DELETE)", 0 ); break; case LDAP_SYNC_MODIFY: - Debug( LDAP_DEBUG_SYNC, "syncrepl_entry: rid %03ld %s\n", - si->si_rid, + Debug( LDAP_DEBUG_SYNC, "syncrepl_entry: %s %s\n", + si->si_ridtxt, "LDAP_RES_SEARCH_ENTRY(LDAP_SYNC_MODIFY)", 0 ); break; default: - Debug( LDAP_DEBUG_ANY, "syncrepl_entry: rid %03ld %s\n", - si->si_rid, + Debug( LDAP_DEBUG_ANY, "syncrepl_entry: %s %s\n", + si->si_ridtxt, "LDAP_RES_SEARCH_ENTRY(UNKNOWN syncstate)", 0 ); } @@ -1681,8 +1754,8 @@ syncrepl_entry( ava.aa_value = *syncUUID; if ( syncuuid_bv ) { - Debug( LDAP_DEBUG_SYNC, "syncrepl_entry: rid %03ld inserted UUID %s\n", - si->si_rid, syncUUID_strrep.bv_val, 0 ); + Debug( LDAP_DEBUG_SYNC, "syncrepl_entry: %s inserted UUID %s\n", + si->si_ridtxt, syncUUID_strrep.bv_val, 0 ); } op->ors_filter = &f; @@ -1720,8 +1793,8 @@ syncrepl_entry( if ( limits_check( op, &rs_search ) == 0 ) { rc = be->be_search( op, &rs_search ); Debug( LDAP_DEBUG_SYNC, - "syncrepl_entry: rid %03ld be_search (%d)\n", - si->si_rid, rc, 0 ); + "syncrepl_entry: %s be_search (%d)\n", + si->si_ridtxt, rc, 0 ); } if ( !BER_BVISNULL( &op->ors_filterstr ) ) { @@ -1733,12 +1806,12 @@ syncrepl_entry( if ( entry && !BER_BVISNULL( &entry->e_name ) ) { Debug( LDAP_DEBUG_SYNC, - "syncrepl_entry: rid %03ld %s\n", - si->si_rid, entry->e_name.bv_val, 0 ); + "syncrepl_entry: %s %s\n", + si->si_ridtxt, entry->e_name.bv_val, 0 ); } else { Debug( LDAP_DEBUG_SYNC, - "syncrepl_entry: rid %03ld %s\n", - si->si_rid, dni.dn.bv_val ? dni.dn.bv_val : "(null)", 0 ); + "syncrepl_entry: %s %s\n", + si->si_ridtxt, dni.dn.bv_val ? dni.dn.bv_val : "(null)", 0 ); } /* Don't save the contextCSN on the inooming context entry, @@ -1785,8 +1858,8 @@ retry_add:; rc = be->be_add( op, &rs_add ); Debug( LDAP_DEBUG_SYNC, - "syncrepl_entry: rid %03ld be_add (%d)\n", - si->si_rid, rc, 0 ); + "syncrepl_entry: %s be_add (%d)\n", + si->si_ridtxt, rc, 0 ); switch ( rs_add.sr_err ) { case LDAP_SUCCESS: if ( op->ora_e == entry ) { @@ -1847,8 +1920,8 @@ retry_add:; default: Debug( LDAP_DEBUG_ANY, - "syncrepl_entry: rid %03ld be_add failed (%d)\n", - si->si_rid, rs_add.sr_err, 0 ); + "syncrepl_entry: %s be_add failed (%d)\n", + si->si_ridtxt, rs_add.sr_err, 0 ); break; } goto done; @@ -1923,8 +1996,8 @@ retry_add:; slap_mods_free( op->orr_modlist, 1 ); Debug( LDAP_DEBUG_SYNC, - "syncrepl_entry: rid %03ld be_modrdn (%d)\n", - si->si_rid, rc, 0 ); + "syncrepl_entry: %s be_modrdn (%d)\n", + si->si_ridtxt, rc, 0 ); goto done; } if ( dni.mods ) { @@ -1935,17 +2008,17 @@ retry_add:; rc = be->be_modify( op, &rs_modify ); slap_mods_free( op->orm_modlist, 1 ); Debug( LDAP_DEBUG_SYNC, - "syncrepl_entry: rid %03ld be_modify (%d)\n", - si->si_rid, rc, 0 ); + "syncrepl_entry: %s be_modify (%d)\n", + si->si_ridtxt, rc, 0 ); if ( rs_modify.sr_err != LDAP_SUCCESS ) { Debug( LDAP_DEBUG_ANY, - "syncrepl_entry: rid %03ld be_modify failed (%d)\n", - si->si_rid, rs_modify.sr_err, 0 ); + "syncrepl_entry: %s be_modify failed (%d)\n", + si->si_ridtxt, rs_modify.sr_err, 0 ); } } else { Debug( LDAP_DEBUG_SYNC, - "syncrepl_entry: rid %03ld entry unchanged, ignored (%s)\n", - si->si_rid, op->o_req_dn.bv_val, 0 ); + "syncrepl_entry: %s entry unchanged, ignored (%s)\n", + si->si_ridtxt, op->o_req_dn.bv_val, 0 ); } goto done; case LDAP_SYNC_DELETE : @@ -1955,8 +2028,8 @@ retry_add:; op->o_tag = LDAP_REQ_DELETE; rc = be->be_delete( op, &rs_delete ); Debug( LDAP_DEBUG_SYNC, - "syncrepl_entry: rid %03ld be_delete (%d)\n", - si->si_rid, rc, 0 ); + "syncrepl_entry: %s be_delete (%d)\n", + si->si_ridtxt, rc, 0 ); while ( rs_delete.sr_err == LDAP_SUCCESS && op->o_delete_glue_parent ) { @@ -1978,7 +2051,7 @@ retry_add:; default : Debug( LDAP_DEBUG_ANY, - "syncrepl_entry: rid %03ld unknown syncstate\n", si->si_rid, 0, 0 ); + "syncrepl_entry: %s unknown syncstate\n", si->si_ridtxt, 0, 0 ); goto done; } @@ -2095,7 +2168,7 @@ syncrepl_del_nonpresent( if ( cookiecsn && !BER_BVISNULL( cookiecsn ) ) { csn = *cookiecsn; } else { - csn = si->si_syncCookie.ctxcsn; + csn = si->si_syncCookie.ctxcsn[0]; } slap_queue_csn( op, &csn ); @@ -2113,8 +2186,8 @@ syncrepl_del_nonpresent( op->o_req_ndn = *np_prev->npe_nname; rc = op->o_bd->be_delete( op, &rs_delete ); Debug( LDAP_DEBUG_SYNC, - "syncrepl_del_nonpresent: rid %03ld be_delete %s (%d)\n", - si->si_rid, op->o_req_dn.bv_val, rc ); + "syncrepl_del_nonpresent: %s be_delete %s (%d)\n", + si->si_ridtxt, op->o_req_dn.bv_val, rc ); if ( rs_delete.sr_err == LDAP_NOT_ALLOWED_ON_NONLEAF ) { Modifications mod1, mod2; @@ -2329,27 +2402,57 @@ syncrepl_updateCookie( struct sync_cookie *syncCookie ) { Backend *be = op->o_bd; - Modifications mod = { { 0 } }; - struct berval vals[ 2 ]; + Modifications mod[2]; + struct berval first = BER_BVNULL; - int rc, flags; + int rc, i, j; slap_callback cb = { NULL }; SlapReply rs_modify = {REP_RESULT}; - mod.sml_op = LDAP_MOD_REPLACE; - mod.sml_desc = slap_schema.si_ad_contextCSN; - mod.sml_type = mod.sml_desc->ad_cname; - mod.sml_values = vals; - vals[0] = syncCookie->ctxcsn; - BER_BVZERO( &vals[1] ); - - slap_queue_csn( op, &syncCookie->ctxcsn ); + mod[0].sml_op = LDAP_MOD_DELETE; + mod[0].sml_desc = slap_schema.si_ad_contextCSN; + mod[0].sml_type = mod[0].sml_desc->ad_cname; + mod[0].sml_values = NULL; + mod[0].sml_nvalues = NULL; + mod[0].sml_next = &mod[1]; + + mod[1].sml_op = LDAP_MOD_ADD; + mod[1].sml_desc = slap_schema.si_ad_contextCSN; + mod[1].sml_type = mod[0].sml_desc->ad_cname; + mod[1].sml_values = NULL; + mod[1].sml_nvalues = NULL; + mod[1].sml_next = NULL; + + ldap_pvt_thread_mutex_lock( &si->si_cookieState->cs_mutex ); + + for ( i=0; inumcsns; i++ ) { + for ( j=0; jsi_cookieState->cs_num; j++ ) { + if ( syncCookie->sids[i] != si->si_cookieState->cs_sids[j] ) + continue; + if ( ber_bvcmp( &syncCookie->ctxcsn[i], + &si->si_cookieState->cs_vals[j] ) > 0 ) { + ber_bvarray_add_x( &mod[0].sml_values, + &si->si_cookieState->cs_vals[j], op->o_tmpmemctx ); + ber_bvarray_add_x( &mod[1].sml_values, + &syncCookie->ctxcsn[i], op->o_tmpmemctx ); + if ( BER_BVISNULL( &first )) + first = syncCookie->ctxcsn[i]; + } + break; + } + /* there was no match for this SID, it's a new CSN */ + if ( j == si->si_cookieState->cs_num ) { + ber_bvarray_add_x( &mod[1].sml_values, + &syncCookie->ctxcsn[i], op->o_tmpmemctx ); + if ( BER_BVISNULL( &first )) + first = syncCookie->ctxcsn[i]; + } + } + slap_queue_csn( op, &first ); op->o_tag = LDAP_REQ_MODIFY; - assert( si->si_rid < 1000 ); - cb.sc_response = null_callback; cb.sc_private = si; @@ -2359,26 +2462,55 @@ syncrepl_updateCookie( /* update contextCSN */ op->o_msgid = SLAP_SYNC_UPDATE_MSGID; - op->orm_modlist = &mod; - flags = SLAP_DBFLAGS( op->o_bd ); - SLAP_DBFLAGS( op->o_bd ) |= SLAP_DBFLAG_NOLASTMOD; + + if ( mod[0].sml_values ) + op->orm_modlist = mod; + else + op->orm_modlist = &mod[1]; + + op->orm_no_opattrs = 1; rc = be->be_modify( op, &rs_modify ); - SLAP_DBFLAGS( op->o_bd ) = flags; op->o_msgid = 0; if ( rs_modify.sr_err == LDAP_SUCCESS ) { slap_sync_cookie_free( &si->si_syncCookie, 0 ); slap_dup_sync_cookie( &si->si_syncCookie, syncCookie ); + /* If we replaced any old values */ + if ( mod[0].sml_values ) { + for ( i=0; !BER_BVISNULL( &mod[0].sml_values[i] ); i++ ) { + for ( j=0; jsi_cookieState->cs_num; j++ ) { + if ( mod[0].sml_values[i].bv_val != + si->si_cookieState->cs_vals[j].bv_val ) + continue; + ber_bvreplace( &si->si_cookieState->cs_vals[j], + &mod[1].sml_values[i] ); + break; + } + } + } else { + /* Else we just added */ + si->si_cookieState->cs_num += syncCookie->numcsns; + value_add( &si->si_cookieState->cs_vals, syncCookie->ctxcsn ); + free( si->si_cookieState->cs_sids ); + si->si_cookieState->cs_sids = slap_parse_csn_sids( + si->si_cookieState->cs_vals, si->si_cookieState->cs_num ); + } + + si->si_cookieState->cs_age++; + si->si_cookieAge = si->si_cookieState->cs_age; } else { Debug( LDAP_DEBUG_ANY, - "syncrepl_updateCookie: rid %03ld be_modify failed (%d)\n", - si->si_rid, rs_modify.sr_err, 0 ); + "syncrepl_updateCookie: %s be_modify failed (%d)\n", + si->si_ridtxt, rs_modify.sr_err, 0 ); } + ldap_pvt_thread_mutex_unlock( &si->si_cookieState->cs_mutex ); slap_graduate_commit_csn( op ); op->o_tmpfree( op->o_csn.bv_val, op->o_tmpmemctx ); BER_BVZERO( &op->o_csn ); - if ( mod.sml_next ) slap_mods_free( mod.sml_next, 1 ); + if ( mod[1].sml_next ) slap_mods_free( mod[1].sml_next, 1 ); + op->o_tmpfree( mod[1].sml_values, op->o_tmpmemctx ); + op->o_tmpfree( mod[0].sml_values, op->o_tmpmemctx ); return rc; } @@ -2662,7 +2794,7 @@ nonpresent_callback( } else if ( rs->sr_type == REP_SEARCH ) { if ( !( si->si_refreshDelete & NP_DELETE_ONE ) ) { - char buf[sizeof("000 not")]; + char buf[sizeof("rid=000 not")]; a = attr_find( rs->sr_entry->e_attrs, slap_schema.si_ad_entryUUID ); @@ -2672,11 +2804,11 @@ nonpresent_callback( } if ( slap_debug & LDAP_DEBUG_SYNC ) { - sprintf( buf, "%03ld %s", si->si_rid, + sprintf( buf, "%s %s", si->si_ridtxt, present_uuid ? "got" : "not" ); } - Debug( LDAP_DEBUG_SYNC, "nonpresent_callback: rid %s UUID %s, dn %s\n", + Debug( LDAP_DEBUG_SYNC, "nonpresent_callback: %s UUID %s, dn %s\n", buf, a ? a->a_vals[0].bv_val : "", rs->sr_entry->e_name.bv_val ); if ( a == NULL ) return 0; @@ -2992,6 +3124,7 @@ parse_syncrepl_line( return -1; } si->si_rid = tmp; + sprintf( si->si_ridtxt, IDSTR "=%03d", si->si_rid ); gots |= GOT_ID; } else if ( !strncasecmp( c->argv[ i ], PROVIDERSTR "=", STRLENOF( PROVIDERSTR "=" ) ) ) @@ -3468,7 +3601,7 @@ add_syncrepl( init_syncrepl( si ); si->si_re = ldap_pvt_runqueue_insert( &slapd_rq, si->si_interval, do_syncrepl, si, "do_syncrepl", - c->be->be_suffix[0].bv_val ); + si->si_ridtxt ); if ( si->si_re ) rc = config_sync_shadow( c ) ? -1 : 0; else @@ -3493,6 +3626,13 @@ add_syncrepl( if ( !si->si_schemachecking ) { SLAP_DBFLAGS(c->be) |= SLAP_DBFLAG_NO_SCHEMA_CHECK; } + if ( c->be->be_syncinfo ) { + si->si_cookieState = c->be->be_syncinfo->si_cookieState; + } else { + si->si_cookieState = ch_calloc( 1, sizeof( cookie_state )); + ldap_pvt_thread_mutex_init( &si->si_cookieState->cs_mutex ); + } + si->si_next = c->be->be_syncinfo; c->be->be_syncinfo = si; return 0; } @@ -3667,20 +3807,43 @@ syncrepl_config( ConfigArgs *c ) if (c->op == SLAP_CONFIG_EMIT) { if ( c->be->be_syncinfo ) { struct berval bv; - syncrepl_unparse( c->be->be_syncinfo, &bv ); - ber_bvarray_add( &c->rvalue_vals, &bv ); + syncinfo_t *si; + + for ( si = c->be->be_syncinfo; si; si=si->si_next ) { + syncrepl_unparse( si, &bv ); + ber_bvarray_add( &c->rvalue_vals, &bv ); + } return 0; } return 1; } else if ( c->op == LDAP_MOD_DELETE ) { + cookie_state *cs = NULL; if ( c->be->be_syncinfo ) { - syncinfo_free( c->be->be_syncinfo ); - c->be->be_syncinfo = NULL; + syncinfo_t *si, **sip; + int i; + + cs = c->be->be_syncinfo->si_cookieState; + for ( sip = &c->be->be_syncinfo, i=0; *sip; i++ ) { + si = *sip; + if ( c->valx == -1 || i == c->valx ) { + *sip = si->si_next; + syncinfo_free( si ); + } else { + sip = &si->si_next; + } + } + } + if ( !c->be->be_syncinfo ) { + SLAP_DBFLAGS( c->be ) &= ~(SLAP_DBFLAG_SHADOW|SLAP_DBFLAG_SYNC_SHADOW); + if ( cs ) { + ber_bvarray_free( cs->cs_vals ); + ldap_pvt_thread_mutex_destroy( &cs->cs_mutex ); + ch_free( cs ); + } } - SLAP_DBFLAGS( c->be ) &= ~(SLAP_DBFLAG_SHADOW|SLAP_DBFLAG_SYNC_SHADOW); return 0; } - if ( SLAP_SHADOW( c->be ) ) { + if ( SLAP_SLURP_SHADOW( c->be ) ) { Debug(LDAP_DEBUG_ANY, "%s: " "syncrepl: database already shadowed.\n", c->log, 0, 0);