]> git.sur5r.net Git - openldap/blobdiff - servers/slapd/overlays/syncprov.c
Fix modify deadlock
[openldap] / servers / slapd / overlays / syncprov.c
index 5847451ea2e7dc4b93dfba5d52afc8400ec9bd55..973ec678d95b437f1e3d564b5b354a31fddf4045 100644 (file)
@@ -2,7 +2,7 @@
 /* syncprov.c - syncrepl provider */
 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
  *
- * Copyright 2004 The OpenLDAP Foundation.
+ * Copyright 2004-2005 The OpenLDAP Foundation.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -56,8 +56,7 @@ typedef struct syncops {
        struct berval   s_base;         /* ndn of search base */
        ID              s_eid;          /* entryID of search base */
        Operation       *s_op;          /* search op */
-       long    s_sid;
-       long    s_rid;
+       int             s_rid;
        struct berval s_filterstr;
        int             s_flags;        /* search status */
        int             s_inuse;        /* reference count */
@@ -84,6 +83,7 @@ typedef struct sync_control {
 #define SLAP_SYNC_REFRESH_AND_PERSIST  (LDAP_SYNC_REFRESH_AND_PERSIST<<SLAP_CONTROL_SHIFT)
 
 #define        PS_IS_REFRESHING        0x01
+#define        PS_IS_DETACHED          0x02
 
 /* Record of which searches matched at premodify step */
 typedef struct syncmatches {
@@ -91,6 +91,23 @@ typedef struct syncmatches {
        syncops *sm_op;
 } syncmatches;
 
+/* Session log data */
+typedef struct slog_entry {
+       struct slog_entry *se_next;
+       struct berval se_uuid;
+       struct berval se_csn;
+       ber_tag_t       se_tag;
+} slog_entry;
+
+typedef struct sessionlog {
+       struct berval   sl_mincsn;
+       int             sl_num;
+       int             sl_size;
+       slog_entry *sl_head;
+       slog_entry *sl_tail;
+       ldap_pvt_thread_mutex_t sl_mutex;
+} sessionlog;
+
 /* The main state for this overlay */
 typedef struct syncprov_info_t {
        syncops         *si_ops;
@@ -100,6 +117,7 @@ typedef struct syncprov_info_t {
        int             si_numops;      /* number of ops since last checkpoint */
        time_t  si_chklast;     /* time of last checkpoint */
        Avlnode *si_mods;       /* entries being modified */
+       sessionlog      *si_logs;
        ldap_pvt_thread_mutex_t si_csn_mutex;
        ldap_pvt_thread_mutex_t si_ops_mutex;
        ldap_pvt_thread_mutex_t si_mods_mutex;
@@ -232,62 +250,6 @@ syncprov_done_ctrl(
        return LDAP_SUCCESS;
 }
 
-#if 0
-/* Generate state based on session log - not implemented yet */
-static int
-syncprov_state_ctrl_from_slog(
-       Operation       *op,
-       SlapReply       *rs,
-       struct slog_entry *slog_e,
-       int                     entry_sync_state,
-       LDAPControl     **ctrls,
-       int                     num_ctrls,
-       int                     send_cookie,
-       struct berval   *cookie)
-{
-       Attribute* a;
-       int ret;
-       int res;
-       const char *text = NULL;
-
-       BerElementBuffer berbuf;
-       BerElement *ber = (BerElement *)&berbuf;
-
-       struct berval entryuuid_bv      = BER_BVNULL;
-
-       ber_init2( ber, NULL, LBER_USE_DER );
-       ber_set_option( ber, LBER_OPT_BER_MEMCTX, &op->o_tmpmemctx );
-
-       ctrls[num_ctrls] = ch_malloc ( sizeof ( LDAPControl ) );
-
-       entryuuid_bv = slog_e->sl_uuid;
-
-       if ( send_cookie && cookie ) {
-               ber_printf( ber, "{eOON}",
-                       entry_sync_state, &entryuuid_bv, cookie );
-       } else {
-               ber_printf( ber, "{eON}",
-                       entry_sync_state, &entryuuid_bv );
-       }
-
-       ctrls[num_ctrls]->ldctl_oid = LDAP_CONTROL_SYNC_STATE;
-       ctrls[num_ctrls]->ldctl_iscritical = (op->o_sync == SLAP_CONTROL_CRITICAL);
-       ret = ber_flatten2( ber, &ctrls[num_ctrls]->ldctl_value, 1 );
-
-       ber_free_buf( ber );
-
-       if ( ret < 0 ) {
-               Debug( LDAP_DEBUG_TRACE,
-                       "slap_build_sync_ctrl: ber_flatten2 failed\n",
-                       0, 0, 0 );
-               send_ldap_error( op, rs, LDAP_OTHER, "internal error" );
-               return ret;
-       }
-
-       return LDAP_SUCCESS;
-}
-#endif
-
 static int
 syncprov_sendinfo(
        Operation       *op,
@@ -452,6 +414,7 @@ syncprov_findbase( Operation *op, fbase_cookie *fc )
        cb.sc_private = fc;
 
        fop.o_sync_mode &= SLAP_CONTROL_MASK;   /* turn off sync mode */
+       fop.o_managedsait = SLAP_CONTROL_CRITICAL;
        fop.o_callback = &cb;
        fop.o_tag = LDAP_REQ_SEARCH;
        fop.ors_scope = LDAP_SCOPE_BASE;
@@ -505,7 +468,7 @@ findmax_cb( Operation *op, SlapReply *rs )
                Attribute *a = attr_find( rs->sr_entry->e_attrs,
                        slap_schema.si_ad_entryCSN );
 
-               if ( a && ber_bvcmp( &a->a_vals[0], maxcsn )) {
+               if ( a && ber_bvcmp( &a->a_vals[0], maxcsn ) > 0 ) {
                        maxcsn->bv_len = a->a_vals[0].bv_len;
                        strcpy( maxcsn->bv_val, a->a_vals[0].bv_val );
                }
@@ -539,11 +502,12 @@ findpres_cb( Operation *op, SlapReply *rs )
 {
        slap_callback *sc = op->o_callback;
        fpres_cookie *pc = sc->sc_private;
+       Attribute *a;
        int ret = SLAP_CB_CONTINUE;
 
-       if ( rs->sr_type == REP_SEARCH ) {
-               Attribute *a = attr_find( rs->sr_entry->e_attrs,
-                       slap_schema.si_ad_entryUUID );
+       switch ( rs->sr_type ) {
+       case REP_SEARCH:
+               a = attr_find( rs->sr_entry->e_attrs, slap_schema.si_ad_entryUUID );
                if ( a ) {
                        pc->uuids[pc->num].bv_val = pc->last;
                        AC_MEMCPY( pc->uuids[pc->num].bv_val, a->a_nvals[0].bv_val,
@@ -553,15 +517,10 @@ findpres_cb( Operation *op, SlapReply *rs )
                        pc->uuids[pc->num].bv_val = NULL;
                }
                ret = LDAP_SUCCESS;
-               if ( pc->num == SLAP_SYNCUUID_SET_SIZE ) {
-                       ret = syncprov_sendinfo( op, rs, LDAP_TAG_SYNC_ID_SET, NULL,
-                               0, pc->uuids, 0 );
-                       pc->uuids[pc->num].bv_val = pc->last;
-                       pc->num = 0;
-                       pc->last = pc->uuids[0].bv_val;
-               }
-
-       } else if ( rs->sr_type == REP_RESULT ) {
+               if ( pc->num != SLAP_SYNCUUID_SET_SIZE )
+                       break;
+               /* FALLTHRU */
+       case REP_RESULT:
                ret = rs->sr_err;
                if ( pc->num ) {
                        ret = syncprov_sendinfo( op, rs, LDAP_TAG_SYNC_ID_SET, NULL,
@@ -570,6 +529,9 @@ findpres_cb( Operation *op, SlapReply *rs )
                        pc->num = 0;
                        pc->last = pc->uuids[0].bv_val;
                }
+               break;
+       default:
+               break;
        }
        return ret;
 }
@@ -587,21 +549,27 @@ syncprov_findcsn( Operation *op, int mode )
        char cbuf[LDAP_LUTIL_CSNSTR_BUFSIZE];
        struct berval fbuf, maxcsn;
        Filter cf, af;
-       AttributeAssertion eq;
+#ifdef LDAP_COMP_MATCH
+       AttributeAssertion eq = { NULL, BER_BVNULL, NULL };
+#else
+       AttributeAssertion eq = { NULL, BER_BVNULL };
+#endif
        int i, rc = LDAP_SUCCESS;
        fpres_cookie pcookie;
-       sync_control *srs;
+       sync_control *srs = NULL;
 
        if ( mode != FIND_MAXCSN ) {
                srs = op->o_controls[slap_cids.sc_LDAPsync];
 
-               if ( srs->sr_state.ctxcsn->bv_len >= LDAP_LUTIL_CSNSTR_BUFSIZE ) {
+               if ( srs->sr_state.ctxcsn.bv_len >= LDAP_LUTIL_CSNSTR_BUFSIZE ) {
                        return LDAP_OTHER;
                }
        }
 
        fop = *op;
        fop.o_sync_mode &= SLAP_CONTROL_MASK;   /* turn off sync_mode */
+       /* We want pure entries, not referrals */
+       fop.o_managedsait = SLAP_CONTROL_CRITICAL;
 
        fbuf.bv_val = buf;
        cf.f_ava = &eq;
@@ -625,12 +593,13 @@ syncprov_findcsn( Operation *op, int mode )
                fop.ors_slimit = SLAP_NO_LIMIT;
                cb.sc_private = &maxcsn;
                cb.sc_response = findmax_cb;
+               strcpy( cbuf, si->si_ctxcsn.bv_val );
                maxcsn.bv_val = cbuf;
-               maxcsn.bv_len = 0;
+               maxcsn.bv_len = si->si_ctxcsn.bv_len;
                break;
        case FIND_CSN:
                cf.f_choice = LDAP_FILTER_LE;
-               cf.f_av_value = *srs->sr_state.ctxcsn;
+               cf.f_av_value = srs->sr_state.ctxcsn;
                fbuf.bv_len = sprintf( buf, "(entryCSN<=%s)",
                        cf.f_av_value.bv_val );
                fop.ors_attrsonly = 1;
@@ -644,15 +613,13 @@ syncprov_findcsn( Operation *op, int mode )
                af.f_next = NULL;
                af.f_and = &cf;
                cf.f_choice = LDAP_FILTER_LE;
-               cf.f_av_value = *srs->sr_state.ctxcsn;
+               cf.f_av_value = srs->sr_state.ctxcsn;
                cf.f_next = op->ors_filter;
                fop.ors_filter = &af;
                filter2bv_x( &fop, fop.ors_filter, &fop.ors_filterstr );
                fop.ors_attrsonly = 0;
                fop.ors_attrs = uuid_anlist;
                fop.ors_slimit = SLAP_NO_LIMIT;
-               /* We want pure entries, not referrals */
-               fop.o_managedsait = SLAP_CONTROL_CRITICAL;
                cb.sc_private = &pcookie;
                cb.sc_response = findpres_cb;
                pcookie.num = 0;
@@ -677,10 +644,8 @@ syncprov_findcsn( Operation *op, int mode )
 
        switch( mode ) {
        case FIND_MAXCSN:
-               if ( maxcsn.bv_len ) {
-                       strcpy( si->si_ctxcsnbuf, maxcsn.bv_val );
-                       si->si_ctxcsn.bv_len = maxcsn.bv_len;
-               }
+               strcpy( si->si_ctxcsnbuf, maxcsn.bv_val );
+               si->si_ctxcsn.bv_len = maxcsn.bv_len;
                break;
        case FIND_CSN:
                /* If matching CSN was not found, invalidate the context. */
@@ -695,7 +660,7 @@ syncprov_findcsn( Operation *op, int mode )
        return rc;
 }
 
-/* Queue a persistent search response if still in Refresh stage */
+/* Queue a persistent search response */
 static int
 syncprov_qresp( opcookie *opc, syncops *so, int mode )
 {
@@ -705,13 +670,18 @@ syncprov_qresp( opcookie *opc, syncops *so, int mode )
                opc->sdn.bv_len + 1 + opc->sndn.bv_len + 1 + opc->sctxcsn.bv_len + 1 );
        sr->s_next = NULL;
        sr->s_dn.bv_val = (char *)(sr + 1);
+       sr->s_dn.bv_len = opc->sdn.bv_len;
        sr->s_mode = mode;
        sr->s_isreference = opc->sreference;
        sr->s_ndn.bv_val = lutil_strcopy( sr->s_dn.bv_val, opc->sdn.bv_val );
+       sr->s_ndn.bv_len = opc->sndn.bv_len;
        *(sr->s_ndn.bv_val++) = '\0';
        sr->s_uuid.bv_val = lutil_strcopy( sr->s_ndn.bv_val, opc->sndn.bv_val );
+       sr->s_uuid.bv_len = opc->suuid.bv_len;
        *(sr->s_uuid.bv_val++) = '\0';
        sr->s_csn.bv_val = lutil_strcopy( sr->s_uuid.bv_val, opc->suuid.bv_val );
+       sr->s_csn.bv_len = opc->sctxcsn.bv_len;
+       strcpy( sr->s_csn.bv_val, opc->sctxcsn.bv_val );
 
        if ( !so->s_res ) {
                so->s_res = sr;
@@ -723,9 +693,58 @@ syncprov_qresp( opcookie *opc, syncops *so, int mode )
        return LDAP_SUCCESS;
 }
 
+/* Play back queued responses */
+static int
+syncprov_sendresp( Operation *op, opcookie *opc, syncops *so, Entry **e, int mode, int queue );
+
+static int
+syncprov_qplay( Operation *op, slap_overinst *on, syncops *so )
+{
+       syncres *sr, *srnext;
+       Entry *e;
+       opcookie opc;
+       int rc;
+
+       opc.son = on;
+       op->o_bd->bd_info = (BackendInfo *)on->on_info;
+       for (sr = so->s_res; sr; sr=srnext) {
+               int rc;
+               srnext = sr->s_next;
+               opc.sdn = sr->s_dn;
+               opc.sndn = sr->s_ndn;
+               opc.suuid = sr->s_uuid;
+               opc.sctxcsn = sr->s_csn;
+               opc.sreference = sr->s_isreference;
+               e = NULL;
+
+               if ( sr->s_mode != LDAP_SYNC_DELETE ) {
+                       rc = be_entry_get_rw( op, &opc.sndn, NULL, NULL, 0, &e );
+                       if ( rc ) {
+                               ch_free( sr );
+                               so->s_res = srnext;
+                               continue;
+                       }
+               }
+               rc = syncprov_sendresp( op, &opc, so, &e, sr->s_mode, 0 );
+
+               if ( e ) {
+                       be_entry_release_rw( op, e, 0 );
+               }
+               if ( rc )
+                       break;
+
+               ch_free( sr );
+               so->s_res = srnext;
+       }
+       op->o_bd->bd_info = (BackendInfo *)on;
+       if ( !so->s_res )
+               so->s_restail = NULL;
+       return rc;
+}
+
 /* Send a persistent search response */
 static int
-syncprov_sendresp( Operation *op, opcookie *opc, syncops *so, Entry *e, int mode, int queue )
+syncprov_sendresp( Operation *op, opcookie *opc, syncops *so, Entry **e, int mode, int queue )
 {
        slap_overinst *on = opc->son;
        syncprov_info_t *si = on->on_bi.bi_private;
@@ -743,17 +762,34 @@ syncprov_sendresp( Operation *op, opcookie *opc, syncops *so, Entry *e, int mode
        sop.o_tmpmemctx = op->o_tmpmemctx;
        sop.o_bd = op->o_bd;
        sop.o_controls = op->o_controls;
+       sop.o_private = op->o_private;
 
-       if ( queue && (so->s_flags & PS_IS_REFRESHING) ) {
+       /* If queueing is allowed */
+       if ( queue ) {
                ldap_pvt_thread_mutex_lock( &so->s_mutex );
-               if ( so->s_flags & PS_IS_REFRESHING )
+               /* If we're still in refresh mode, must queue */
+               if (so->s_flags & PS_IS_REFRESHING) {
+                       return syncprov_qresp( opc, so, mode );
+               }
+               /* If connection is free but queue is non-empty,
+                * try to flush the queue.
+                */
+               if ( so->s_res ) {
+                       rs.sr_err = syncprov_qplay( &sop, on, so );
+               }
+               /* If the connection is busy, must queue */
+               if ( sop.o_conn->c_writewaiter || rs.sr_err == LDAP_BUSY ) {
                        return syncprov_qresp( opc, so, mode );
+               }
                ldap_pvt_thread_mutex_unlock( &so->s_mutex );
+       } else {
+               /* Queueing not allowed and conn is busy, give up */
+               if ( sop.o_conn->c_writewaiter )
+                       return LDAP_BUSY;
        }
 
        ctrls[1] = NULL;
-       slap_compose_sync_cookie( op, &cookie, &opc->sctxcsn,
-               so->s_sid, so->s_rid );
+       slap_compose_sync_cookie( op, &cookie, &opc->sctxcsn, so->s_rid );
 
        e_uuid.e_attrs = &a_uuid;
        a_uuid.a_desc = slap_schema.si_ad_entryUUID;
@@ -761,20 +797,30 @@ syncprov_sendresp( Operation *op, opcookie *opc, syncops *so, Entry *e, int mode
        rs.sr_err = syncprov_state_ctrl( &sop, &rs, &e_uuid,
                mode, ctrls, 0, 1, &cookie );
 
-       rs.sr_entry = e;
        rs.sr_ctrls = ctrls;
+       op->o_bd->bd_info = (BackendInfo *)on->on_info;
        switch( mode ) {
        case LDAP_SYNC_ADD:
+               rs.sr_entry = *e;
+               if ( rs.sr_entry->e_private )
+                       rs.sr_flags = REP_ENTRY_MUSTRELEASE;
                if ( opc->sreference ) {
-                       rs.sr_ref = get_entry_referrals( &sop, e );
+                       rs.sr_ref = get_entry_referrals( &sop, rs.sr_entry );
                        send_search_reference( &sop, &rs );
                        ber_bvarray_free( rs.sr_ref );
+                       if ( !rs.sr_entry )
+                               *e = NULL;
                        break;
                }
                /* fallthru */
        case LDAP_SYNC_MODIFY:
+               rs.sr_entry = *e;
+               if ( rs.sr_entry->e_private )
+                       rs.sr_flags = REP_ENTRY_MUSTRELEASE;
                rs.sr_attrs = sop.ors_attrs;
                send_search_entry( &sop, &rs );
+               if ( !rs.sr_entry )
+                       *e = NULL;
                break;
        case LDAP_SYNC_DELETE:
                e_uuid.e_attrs = NULL;
@@ -793,6 +839,7 @@ syncprov_sendresp( Operation *op, opcookie *opc, syncops *so, Entry *e, int mode
                assert(0);
        }
        op->o_tmpfree( rs.sr_ctrls[0], op->o_tmpmemctx );
+       op->o_private = sop.o_private;
        rs.sr_ctrls = NULL;
        return rs.sr_err;
 }
@@ -801,6 +848,7 @@ static void
 syncprov_free_syncop( syncops *so )
 {
        syncres *sr, *srnext;
+       GroupAssertion *ga, *gnext;
 
        ldap_pvt_thread_mutex_lock( &so->s_mutex );
        so->s_inuse--;
@@ -809,8 +857,14 @@ syncprov_free_syncop( syncops *so )
                return;
        }
        ldap_pvt_thread_mutex_unlock( &so->s_mutex );
-       filter_free( so->s_op->ors_filter );
-       ch_free( so->s_op );
+       if ( so->s_flags & PS_IS_DETACHED ) {
+               filter_free( so->s_op->ors_filter );
+               for ( ga = so->s_op->o_groups; ga; ga=gnext ) {
+                       gnext = ga->ga_next;
+                       ch_free( ga );
+               }
+               ch_free( so->s_op );
+       }
        ch_free( so->s_base.bv_val );
        for ( sr=so->s_res; sr; sr=srnext ) {
                srnext = sr->s_next;
@@ -821,13 +875,31 @@ syncprov_free_syncop( syncops *so )
 }
 
 static int
-syncprov_drop_psearch( syncops *so )
+syncprov_drop_psearch( syncops *so, int lock )
 {
-       ldap_pvt_thread_mutex_lock( &so->s_op->o_conn->c_mutex );
-       so->s_op->o_conn->c_n_ops_executing--;
-       so->s_op->o_conn->c_n_ops_completed++;
-       ldap_pvt_thread_mutex_unlock( &so->s_op->o_conn->c_mutex );
+       if ( so->s_flags & PS_IS_DETACHED ) {
+               if ( lock )
+                       ldap_pvt_thread_mutex_lock( &so->s_op->o_conn->c_mutex );
+               so->s_op->o_conn->c_n_ops_executing--;
+               so->s_op->o_conn->c_n_ops_completed++;
+               LDAP_STAILQ_REMOVE( &so->s_op->o_conn->c_ops, so->s_op, slap_op,
+                       o_next );
+               if ( lock )
+                       ldap_pvt_thread_mutex_unlock( &so->s_op->o_conn->c_mutex );
+       }
        syncprov_free_syncop( so );
+
+       return 0;
+}
+
+static int
+syncprov_ab_cleanup( Operation *op, SlapReply *rs )
+{
+       slap_callback *sc = op->o_callback;
+       op->o_callback = sc->sc_next;
+       syncprov_drop_psearch( op->o_callback->sc_private, 0 );
+       op->o_tmpfree( sc, op->o_tmpmemctx );
+       return 0;
 }
 
 static int
@@ -851,10 +923,19 @@ syncprov_op_abandon( Operation *op, SlapReply *rs )
        if ( so ) {
                /* Is this really a Cancel exop? */
                if ( op->o_tag != LDAP_REQ_ABANDON ) {
+                       so->s_op->o_cancel = SLAP_CANCEL_ACK;
                        rs->sr_err = LDAP_CANCELLED;
                        send_ldap_result( so->s_op, rs );
+                       if ( so->s_flags & PS_IS_DETACHED ) {
+                               slap_callback *cb;
+                               cb = op->o_tmpcalloc( 1, sizeof(slap_callback), op->o_tmpmemctx );
+                               cb->sc_cleanup = syncprov_ab_cleanup;
+                               cb->sc_next = op->o_callback;
+                               cb->sc_private = so;
+                               return SLAP_CB_CONTINUE;
+                       }
                }
-               syncprov_drop_psearch( so );
+               syncprov_drop_psearch( so, 0 );
        }
        return SLAP_CB_CONTINUE;
 }
@@ -893,10 +974,6 @@ syncprov_matchops( Operation *op, opcookie *opc, int saveit )
                e = op->ora_e;
        }
 
-       /* Never replicate these */
-       if ( is_entry_syncConsumerSubentry( e )) {
-               goto done;
-       }
        if ( saveit ) {
                ber_dupbv_x( &opc->sdn, &e->e_name, op->o_tmpmemctx );
                ber_dupbv_x( &opc->sndn, &e->e_nname, op->o_tmpmemctx );
@@ -928,7 +1005,7 @@ syncprov_matchops( Operation *op, opcookie *opc, int saveit )
                        send_ldap_error( ss->s_op, &rs, LDAP_SYNC_REFRESH_REQUIRED,
                                "search base has changed" );
                        sprev->s_next = snext;
-                       syncprov_drop_psearch( ss );
+                       syncprov_drop_psearch( ss, 1 );
                        continue;
                }
 
@@ -957,19 +1034,25 @@ syncprov_matchops( Operation *op, opcookie *opc, int saveit )
                                opc->smatches = sm;
                        } else {
                                /* if found send UPDATE else send ADD */
-                               syncprov_sendresp( op, opc, ss, e,
+                               ss->s_inuse++;
+                               ldap_pvt_thread_mutex_unlock( &si->si_ops_mutex );
+                               syncprov_sendresp( op, opc, ss, &e,
                                        found ? LDAP_SYNC_MODIFY : LDAP_SYNC_ADD, 1 );
+                               ldap_pvt_thread_mutex_lock( &si->si_ops_mutex );
+                               ss->s_inuse--;
                        }
                } else if ( !saveit && found ) {
                        /* send DELETE */
+                       ldap_pvt_thread_mutex_unlock( &si->si_ops_mutex );
                        syncprov_sendresp( op, opc, ss, NULL, LDAP_SYNC_DELETE, 1 );
+                       ldap_pvt_thread_mutex_lock( &si->si_ops_mutex );
                }
        }
        ldap_pvt_thread_mutex_unlock( &si->si_ops_mutex );
 done:
-       if ( op->o_tag != LDAP_REQ_ADD ) {
+       if ( op->o_tag != LDAP_REQ_ADD && e ) {
                op->o_bd->bd_info = (BackendInfo *)on->on_info;
-               be_entry_release_r( op, e );
+               be_entry_release_rw( op, e, 0 );
                op->o_bd->bd_info = (BackendInfo *)on;
        }
        if ( freefdn ) {
@@ -997,7 +1080,6 @@ syncprov_op_cleanup( Operation *op, SlapReply *rs )
        mtdummy.mt_op = op;
        ldap_pvt_thread_mutex_lock( &si->si_mods_mutex );
        mt = avl_find( si->si_mods, &mtdummy, sp_avl_cmp );
-       ldap_pvt_thread_mutex_unlock( &si->si_mods_mutex );
        if ( mt ) {
                modinst *mi = mt->mt_mods;
 
@@ -1008,14 +1090,13 @@ syncprov_op_cleanup( Operation *op, SlapReply *rs )
                        mt->mt_op = mt->mt_mods->mi_op;
                        ldap_pvt_thread_mutex_unlock( &mt->mt_mutex );
                } else {
-                       ldap_pvt_thread_mutex_lock( &si->si_mods_mutex );
                        avl_delete( &si->si_mods, mt, sp_avl_cmp );
-                       ldap_pvt_thread_mutex_unlock( &si->si_mods_mutex );
                        ldap_pvt_thread_mutex_unlock( &mt->mt_mutex );
                        ldap_pvt_thread_mutex_destroy( &mt->mt_mutex );
                        ch_free( mt );
                }
        }
+       ldap_pvt_thread_mutex_unlock( &si->si_mods_mutex );
        if ( !BER_BVISNULL( &opc->suuid ))
                op->o_tmpfree( opc->suuid.bv_val, op->o_tmpmemctx );
        if ( !BER_BVISNULL( &opc->sndn ))
@@ -1024,6 +1105,8 @@ syncprov_op_cleanup( Operation *op, SlapReply *rs )
                op->o_tmpfree( opc->sdn.bv_val, op->o_tmpmemctx );
        op->o_callback = cb->sc_next;
        op->o_tmpfree(cb, op->o_tmpmemctx);
+
+       return 0;
 }
 
 static void
@@ -1033,8 +1116,8 @@ syncprov_checkpoint( Operation *op, SlapReply *rs, slap_overinst *on )
        Modifications mod;
        Operation opm;
        struct berval bv[2];
-       BackendInfo *orig;
        slap_callback cb = {0};
+       int manage = get_manageDSAit(op);
 
        mod.sml_values = bv;
        bv[1].bv_val = NULL;
@@ -1051,9 +1134,199 @@ syncprov_checkpoint( Operation *op, SlapReply *rs, slap_overinst *on )
        opm.orm_modlist = &mod;
        opm.o_req_dn = op->o_bd->be_suffix[0];
        opm.o_req_ndn = op->o_bd->be_nsuffix[0];
-       orig = opm.o_bd->bd_info;
        opm.o_bd->bd_info = on->on_info->oi_orig;
+       opm.o_managedsait = SLAP_CONTROL_NONCRITICAL;
        opm.o_bd->be_modify( &opm, rs );
+       opm.o_managedsait = manage;
+}
+
+static void
+syncprov_add_slog( Operation *op, struct berval *csn )
+{
+       opcookie *opc = op->o_callback->sc_private;
+       slap_overinst *on = opc->son;
+       syncprov_info_t         *si = on->on_bi.bi_private;
+       sessionlog *sl;
+       slog_entry *se;
+
+       sl = si->si_logs;
+       {
+               /* Allocate a record. UUIDs are not NUL-terminated. */
+               se = ch_malloc( sizeof( slog_entry ) + opc->suuid.bv_len + 
+                       csn->bv_len + 1 );
+               se->se_next = NULL;
+               se->se_tag = op->o_tag;
+
+               se->se_uuid.bv_val = (char *)(se+1);
+               se->se_csn.bv_val = se->se_uuid.bv_val + opc->suuid.bv_len + 1;
+               AC_MEMCPY( se->se_uuid.bv_val, opc->suuid.bv_val, opc->suuid.bv_len );
+               se->se_uuid.bv_len = opc->suuid.bv_len;
+
+               AC_MEMCPY( se->se_csn.bv_val, csn->bv_val, csn->bv_len );
+               se->se_csn.bv_val[csn->bv_len] = '\0';
+               se->se_csn.bv_len = csn->bv_len;
+
+               ldap_pvt_thread_mutex_lock( &sl->sl_mutex );
+               if ( sl->sl_head ) {
+                       sl->sl_tail->se_next = se;
+               } else {
+                       sl->sl_head = se;
+               }
+               sl->sl_tail = se;
+               sl->sl_num++;
+               while ( sl->sl_num > sl->sl_size ) {
+                       se = sl->sl_head;
+                       sl->sl_head = se->se_next;
+                       strcpy( sl->sl_mincsn.bv_val, se->se_csn.bv_val );
+                       sl->sl_mincsn.bv_len = se->se_csn.bv_len;
+                       ch_free( se );
+                       sl->sl_num--;
+                       if ( !sl->sl_head ) {
+                               sl->sl_tail = NULL;
+                       }
+               }
+               ldap_pvt_thread_mutex_unlock( &sl->sl_mutex );
+       }
+}
+
+/* Just set a flag if we found the matching entry */
+static int
+playlog_cb( Operation *op, SlapReply *rs )
+{
+       if ( rs->sr_type == REP_SEARCH ) {
+               op->o_callback->sc_private = (void *)1;
+       }
+       return rs->sr_err;
+}
+
+/* enter with sl->sl_mutex locked, release before returning */
+static void
+syncprov_playlog( Operation *op, SlapReply *rs, sessionlog *sl,
+       struct berval *oldcsn, struct berval *ctxcsn )
+{
+       slap_overinst           *on = (slap_overinst *)op->o_bd->bd_info;
+       syncprov_info_t         *si = on->on_bi.bi_private;
+       slog_entry *se;
+       int i, j, ndel, num, nmods, mmods;
+       BerVarray uuids;
+
+       if ( !sl->sl_num ) {
+               ldap_pvt_thread_mutex_unlock( &sl->sl_mutex );
+               return;
+       }
+
+       num = sl->sl_num;
+       i = 0;
+       nmods = 0;
+
+       uuids = op->o_tmpalloc( (num+1) * sizeof( struct berval ) +
+               num * UUID_LEN, op->o_tmpmemctx );
+
+       uuids[0].bv_val = (char *)(uuids + num + 1);
+
+       /* Make a copy of the relevant UUIDs. Put the Deletes up front
+        * and everything else at the end. Do this first so we can
+        * unlock the list mutex.
+        */
+       for ( se=sl->sl_head; se; se=se->se_next ) {
+               if ( ber_bvcmp( &se->se_csn, oldcsn ) < 0 ) continue;
+               if ( ber_bvcmp( &se->se_csn, ctxcsn ) > 0 ) break;
+               if ( se->se_tag == LDAP_REQ_DELETE ) {
+                       j = i;
+                       i++;
+               } else {
+                       nmods++;
+                       j = num - nmods;
+               }
+               uuids[j].bv_val = uuids[0].bv_val + (j * UUID_LEN);
+               AC_MEMCPY(uuids[j].bv_val, se->se_uuid.bv_val, UUID_LEN);
+               uuids[j].bv_len = UUID_LEN;
+       }
+       ldap_pvt_thread_mutex_unlock( &sl->sl_mutex );
+
+       ndel = i;
+
+       /* Zero out unused slots */
+       for ( i=ndel; i < num - nmods; i++ )
+               uuids[i].bv_len = 0;
+
+       /* Mods must be validated to see if they belong in this delete set.
+        */
+
+       mmods = nmods;
+       /* Strip any duplicates */
+       for ( i=0; i<nmods; i++ ) {
+               for ( j=0; j<ndel; j++ ) {
+                       if ( bvmatch( &uuids[j], &uuids[num - 1 - i] )) {
+                               uuids[num - 1 - i].bv_len = 0;
+                               mmods --;
+                               break;
+                       }
+               }
+               if ( uuids[num - 1 - i].bv_len == 0 ) continue;
+               for ( j=0; j<i; j++ ) {
+                       if ( bvmatch( &uuids[num - 1 - j], &uuids[num - 1 - i] )) {
+                               uuids[num - 1 - i].bv_len = 0;
+                               mmods --;
+                               break;
+                       }
+               }
+       }
+
+       if ( mmods ) {
+               Operation fop;
+               SlapReply frs = { REP_RESULT };
+               int rc;
+               Filter mf, af;
+#ifdef LDAP_COMP_MATCH
+               AttributeAssertion eq = { NULL, BER_BVNULL, NULL };
+#else
+               AttributeAssertion eq;
+#endif
+               slap_callback cb = {0};
+
+               fop = *op;
+
+               fop.o_sync_mode = 0;
+               fop.o_callback = &cb;
+               fop.ors_limit = NULL;
+               fop.ors_tlimit = SLAP_NO_LIMIT;
+               fop.ors_attrs = slap_anlist_all_attributes;
+               fop.ors_attrsonly = 0;
+               fop.o_managedsait = SLAP_CONTROL_CRITICAL;
+
+               af.f_choice = LDAP_FILTER_AND;
+               af.f_next = NULL;
+               af.f_and = &mf;
+               mf.f_choice = LDAP_FILTER_EQUALITY;
+               mf.f_ava = &eq;
+               mf.f_av_desc = slap_schema.si_ad_entryUUID;
+               mf.f_next = fop.ors_filter;
+
+               fop.ors_filter = &af;
+
+               cb.sc_response = playlog_cb;
+               fop.o_bd->bd_info = on->on_info->oi_orig;
+
+               for ( i=ndel; i<num; i++ ) {
+                       if ( uuids[i].bv_len == 0 ) continue;
+
+                       mf.f_av_value = uuids[i];
+                       cb.sc_private = NULL;
+                       fop.ors_slimit = 1;
+                       rc = fop.o_bd->be_search( &fop, &frs );
+
+                       /* If entry was not found, add to delete list */
+                       if ( !cb.sc_private ) {
+                               uuids[ndel++] = uuids[i];
+                       }
+               }
+               fop.o_bd->bd_info = (BackendInfo *)on;
+       }
+       if ( ndel ) {
+               uuids[ndel].bv_val = NULL;
+               syncprov_sendinfo( op, rs, LDAP_TAG_SYNC_ID_SET, NULL, 0, uuids, 1 );
+       }
 }
 
 static int
@@ -1066,13 +1339,13 @@ syncprov_op_response( Operation *op, SlapReply *rs )
 
        if ( rs->sr_err == LDAP_SUCCESS )
        {
-               struct berval maxcsn;
+               struct berval maxcsn = BER_BVNULL, curcsn = BER_BVNULL;
                char cbuf[LDAP_LUTIL_CSNSTR_BUFSIZE];
 
                /* Update our context CSN */
                cbuf[0] = '\0';
                ldap_pvt_thread_mutex_lock( &si->si_csn_mutex );
-               slap_get_commit_csn( op, &maxcsn );
+               slap_get_commit_csn( op, &maxcsn, &curcsn );
                if ( !BER_BVISNULL( &maxcsn ) ) {
                        strcpy( cbuf, maxcsn.bv_val );
                        if ( ber_bvcmp( &maxcsn, &si->si_ctxcsn ) > 0 ) {
@@ -1081,6 +1354,13 @@ syncprov_op_response( Operation *op, SlapReply *rs )
                        }
                }
 
+               /* Don't do any processing for consumer contextCSN updates */
+               if ( SLAP_SYNC_SHADOW( op->o_bd ) && 
+                       op->o_msgid == SLAP_SYNC_UPDATE_MSGID ) {
+                       ldap_pvt_thread_mutex_unlock( &si->si_csn_mutex );
+                       return SLAP_CB_CONTINUE;
+               }
+
                si->si_numops++;
                if ( si->si_chkops || si->si_chktime ) {
                        int do_check=0;
@@ -1115,18 +1395,21 @@ syncprov_op_response( Operation *op, SlapReply *rs )
                                /* for each match in opc->smatches:
                                 *   send DELETE msg
                                 */
-                               ldap_pvt_thread_mutex_lock( &si->si_ops_mutex );
                                for ( sm = opc->smatches; sm; sm=sm->sm_next ) {
                                        if ( sm->sm_op->s_op->o_abandon )
                                                continue;
                                        syncprov_sendresp( op, opc, sm->sm_op, NULL,
                                                LDAP_SYNC_DELETE, 1 );
                                }
-                               ldap_pvt_thread_mutex_unlock( &si->si_ops_mutex );
                                break;
                        }
                }
 
+               /* Add any log records */
+               if ( si->si_logs && op->o_tag != LDAP_REQ_ADD ) {
+                       syncprov_add_slog( op, &curcsn );
+               }
+
        }
        return SLAP_CB_CONTINUE;
 }
@@ -1243,6 +1526,18 @@ syncprov_op_mod( Operation *op, SlapReply *rs )
                                ldap_pvt_thread_mutex_unlock( &mt->mt_mutex );
                                ldap_pvt_thread_yield();
                                ldap_pvt_thread_mutex_lock( &mt->mt_mutex );
+
+                               /* clean up if the caller is giving up */
+                               if ( op->o_abandon ) {
+                                       modinst *m2;
+                                       for ( m2 = mt->mt_mods; m2->mi_next != mi;
+                                               m2 = m2->mi_next );
+                                       m2->mi_next = mi->mi_next;
+                                       if ( mt->mt_tail == mi ) mt->mt_tail = m2;
+                                       op->o_tmpfree( cb, op->o_tmpmemctx );
+                                       ldap_pvt_thread_mutex_unlock( &mt->mt_mutex );
+                                       return SLAPD_ABANDON;
+                               }
                        }
                        ldap_pvt_thread_mutex_unlock( &mt->mt_mutex );
                } else {
@@ -1255,11 +1550,11 @@ syncprov_op_mod( Operation *op, SlapReply *rs )
                        avl_insert( &si->si_mods, mt, sp_avl_cmp, avl_dup_error );
                        ldap_pvt_thread_mutex_unlock( &si->si_mods_mutex );
                }
-
-               if ( op->o_tag != LDAP_REQ_ADD )
-                       syncprov_matchops( op, opc, 1 );
        }
 
+       if (( si->si_ops || si->si_logs ) && op->o_tag != LDAP_REQ_ADD )
+               syncprov_matchops( op, opc, 1 );
+
        return SLAP_CB_CONTINUE;
 }
 
@@ -1275,6 +1570,7 @@ syncprov_op_extended( Operation *op, SlapReply *rs )
 typedef struct searchstate {
        slap_overinst *ss_on;
        syncops *ss_so;
+       int ss_present;
 } searchstate;
 
 static int
@@ -1295,6 +1591,7 @@ syncprov_detach_op( Operation *op, syncops *so )
        int i, alen = 0;
        size_t size;
        char *ptr;
+       GroupAssertion *g1, *g2;
 
        /* count the search attrs */
        for (i=0; op->ors_attrs && !BER_BVISNULL( &op->ors_attrs[i].an_name ); i++) {
@@ -1307,10 +1604,16 @@ syncprov_detach_op( Operation *op, syncops *so )
                op->o_req_ndn.bv_len + 1 +
                op->o_ndn.bv_len + 1 +
                so->s_filterstr.bv_len + 1;
-       op2 = (Operation *)ch_malloc( size );
-       *op2 = *op;
+       op2 = (Operation *)ch_calloc( 1, size );
        op2->o_hdr = (Opheader *)(op2+1);
+
+       /* Copy the fields we care about explicitly, leave the rest alone */
        *op2->o_hdr = *op->o_hdr;
+       op2->o_tag = op->o_tag;
+       op2->o_time = op->o_time;
+       op2->o_bd = op->o_bd;
+       op2->o_request = op->o_request;
+
        if ( i ) {
                op2->ors_attrs = (AttributeName *)(op2->o_hdr + 1);
                ptr = (char *)(op2->ors_attrs+i+1);
@@ -1323,25 +1626,38 @@ syncprov_detach_op( Operation *op, syncops *so )
        } else {
                ptr = (char *)(op2->o_hdr + 1);
        }
+       op2->o_authz = op->o_authz;
        op2->o_ndn.bv_val = ptr;
        ptr = lutil_strcopy(ptr, op->o_ndn.bv_val) + 1;
        op2->o_dn = op2->o_ndn;
+       op2->o_req_dn.bv_len = op->o_req_dn.bv_len;
        op2->o_req_dn.bv_val = ptr;
        ptr = lutil_strcopy(ptr, op->o_req_dn.bv_val) + 1;
+       op2->o_req_ndn.bv_len = op->o_req_ndn.bv_len;
        op2->o_req_ndn.bv_val = ptr;
        ptr = lutil_strcopy(ptr, op->o_req_ndn.bv_val) + 1;
        op2->ors_filterstr.bv_val = ptr;
        strcpy( ptr, so->s_filterstr.bv_val );
        op2->ors_filterstr.bv_len = so->s_filterstr.bv_len;
        op2->ors_filter = str2filter( ptr );
-       op2->o_controls = NULL;
-       op2->o_callback = NULL;
        so->s_op = op2;
 
-       /* Increment number of ops so that idletimeout ignores us */
+       /* Copy any cached group ACLs individually */
+       op2->o_groups = NULL;
+       for ( g1=op->o_groups; g1; g1=g1->ga_next ) {
+               g2 = ch_malloc( sizeof(GroupAssertion) + g1->ga_len );
+               *g2 = *g1;
+               strcpy( g2->ga_ndn, g1->ga_ndn );
+               g2->ga_next = op2->o_groups;
+               op2->o_groups = g2;
+       }
+
+       /* Add op2 to conn so abandon will find us */
        ldap_pvt_thread_mutex_lock( &op->o_conn->c_mutex );
        op->o_conn->c_n_ops_executing++;
        op->o_conn->c_n_ops_completed--;
+       LDAP_STAILQ_INSERT_TAIL( &op->o_conn->c_ops, op2, o_next );
+       so->s_flags |= PS_IS_DETACHED;
        ldap_pvt_thread_mutex_unlock( &op->o_conn->c_mutex );
 }
 
@@ -1365,11 +1681,12 @@ syncprov_search_response( Operation *op, SlapReply *rs )
                        Debug( LDAP_DEBUG_ANY, "bogus referral in context\n",0,0,0 );
                        return SLAP_CB_CONTINUE;
                }
-               if ( srs->sr_state.ctxcsn ) {
+               if ( !BER_BVISNULL( &srs->sr_state.ctxcsn )) {
                        Attribute *a = attr_find( rs->sr_entry->e_attrs,
                                slap_schema.si_ad_entryCSN );
+                       
                        /* Don't send the ctx entry twice */
-                       if ( bvmatch( &a->a_nvals[0], srs->sr_state.ctxcsn ))
+                       if ( a && bvmatch( &a->a_nvals[0], &srs->sr_state.ctxcsn ) )
                                return LDAP_SUCCESS;
                }
                rs->sr_ctrls = op->o_tmpalloc( sizeof(LDAPControl *)*2,
@@ -1382,7 +1699,7 @@ syncprov_search_response( Operation *op, SlapReply *rs )
 
                slap_compose_sync_cookie( op, &cookie,
                        &op->ors_filter->f_and->f_ava->aa_value,
-                       srs->sr_state.sid, srs->sr_state.rid );
+                       srs->sr_state.rid );
 
                /* Is this a regular refresh? */
                if ( !ss->ss_so ) {
@@ -1390,50 +1707,19 @@ syncprov_search_response( Operation *op, SlapReply *rs )
                                op->o_tmpmemctx );
                        rs->sr_ctrls[1] = NULL;
                        rs->sr_err = syncprov_done_ctrl( op, rs, rs->sr_ctrls,
-                               0, 1, &cookie, LDAP_SYNC_REFRESH_PRESENTS );
+                               0, 1, &cookie, ss->ss_present ?  LDAP_SYNC_REFRESH_PRESENTS :
+                                       LDAP_SYNC_REFRESH_DELETES );
                } else {
                        int locked = 0;
                /* It's RefreshAndPersist, transition to Persist phase */
-                       syncprov_sendinfo( op, rs, rs->sr_nentries ?
+                       syncprov_sendinfo( op, rs, ( ss->ss_present && rs->sr_nentries ) ?
                                LDAP_TAG_SYNC_REFRESH_PRESENT : LDAP_TAG_SYNC_REFRESH_DELETE,
                                &cookie, 1, NULL, 0 );
                        /* Flush any queued persist messages */
                        if ( ss->ss_so->s_res ) {
-                               syncres *sr, *srnext;
-                               Entry *e;
-                               opcookie opc;
-
-                               opc.son = on;
                                ldap_pvt_thread_mutex_lock( &ss->ss_so->s_mutex );
                                locked = 1;
-                               for (sr = ss->ss_so->s_res; sr; sr=srnext) {
-                                       int rc = LDAP_SUCCESS;
-                                       srnext = sr->s_next;
-                                       opc.sdn = sr->s_dn;
-                                       opc.sndn = sr->s_ndn;
-                                       opc.suuid = sr->s_uuid;
-                                       opc.sctxcsn = sr->s_csn;
-                                       opc.sreference = sr->s_isreference;
-                                       e = NULL;
-
-                                       if ( sr->s_mode != LDAP_SYNC_DELETE ) {
-                                               op->o_bd->bd_info = (BackendInfo *)on->on_info;
-                                               rc = be_entry_get_rw( op, &opc.sndn, NULL, NULL, 0, &e );
-                                               op->o_bd->bd_info = (BackendInfo *)on;
-                                       }
-                                       if ( rc == LDAP_SUCCESS )
-                                               syncprov_sendresp( op, &opc, ss->ss_so, e,
-                                                       sr->s_mode, 0 );
-
-                                       if ( e ) {
-                                               op->o_bd->bd_info = (BackendInfo *)on->on_info;
-                                               be_entry_release_r( op, e );
-                                               op->o_bd->bd_info = (BackendInfo *)on;
-                                       }
-                                       ch_free( sr );
-                               }
-                               ss->ss_so->s_res = NULL;
-                               ss->ss_so->s_restail = NULL;
+                               syncprov_qplay( op, on, ss->ss_so );
                        }
 
                        /* Turn off the refreshing flag */
@@ -1457,11 +1743,13 @@ syncprov_op_search( Operation *op, SlapReply *rs )
        slap_overinst           *on = (slap_overinst *)op->o_bd->bd_info;
        syncprov_info_t         *si = (syncprov_info_t *)on->on_bi.bi_private;
        slap_callback   *cb;
-       int gotstate = 0, nochange = 0;
+       int gotstate = 0, nochange = 0, do_present = 1;
        Filter *fand, *fava;
        syncops *sop = NULL;
        searchstate *ss;
        sync_control *srs;
+       struct berval ctxcsn;
+       char csnbuf[LDAP_LUTIL_CSNSTR_BUFSIZE];
 
        if ( !(op->o_sync_mode & SLAP_SYNC_REFRESH) ) return SLAP_CB_CONTINUE;
 
@@ -1471,6 +1759,7 @@ syncprov_op_search( Operation *op, SlapReply *rs )
        }
 
        srs = op->o_controls[slap_cids.sc_LDAPsync];
+       op->o_managedsait = SLAP_CONTROL_NONCRITICAL;
 
        /* If this is a persistent search, set it up right away */
        if ( op->o_sync_mode & SLAP_SYNC_PERSIST ) {
@@ -1499,25 +1788,30 @@ syncprov_op_search( Operation *op, SlapReply *rs )
                sop = ch_malloc( sizeof( syncops ));
                *sop = so;
                ldap_pvt_thread_mutex_init( &sop->s_mutex );
-               ldap_pvt_thread_mutex_lock( &si->si_ops_mutex );
-               sop->s_sid = srs->sr_state.sid;
                sop->s_rid = srs->sr_state.rid;
-               sop->s_next = si->si_ops;
                sop->s_inuse = 1;
+
+               ldap_pvt_thread_mutex_lock( &si->si_ops_mutex );
+               sop->s_next = si->si_ops;
                si->si_ops = sop;
                ldap_pvt_thread_mutex_unlock( &si->si_ops_mutex );
        }
 
-       /* If we have a cookie, handle the PRESENT lookups
-        */
-       if ( srs->sr_state.ctxcsn ) {
-               /* Is the CSN in a valid format? */
-               if ( srs->sr_state.ctxcsn->bv_len >= LDAP_LUTIL_CSNSTR_BUFSIZE ) {
-                       send_ldap_error( op, rs, LDAP_OTHER, "invalid sync cookie" );
-                       return rs->sr_err;
-               }
+       /* snapshot the ctxcsn */
+       ldap_pvt_thread_mutex_lock( &si->si_csn_mutex );
+       strcpy( csnbuf, si->si_ctxcsnbuf );
+       ctxcsn.bv_len = si->si_ctxcsn.bv_len;
+       ldap_pvt_thread_mutex_unlock( &si->si_csn_mutex );
+       ctxcsn.bv_val = csnbuf;
+       
+       /* If we have a cookie, handle the PRESENT lookups */
+       if ( !BER_BVISNULL( &srs->sr_state.ctxcsn )) {
+               sessionlog *sl;
+
+               /* The cookie was validated when it was parsed, just use it */
+
                /* If just Refreshing and nothing has changed, shortcut it */
-               if ( bvmatch( srs->sr_state.ctxcsn, &si->si_ctxcsn )) {
+               if ( bvmatch( &srs->sr_state.ctxcsn, &ctxcsn )) {
                        nochange = 1;
                        if ( !(op->o_sync_mode & SLAP_SYNC_PERSIST) ) {
                                LDAPControl     *ctrls[2];
@@ -1534,6 +1828,18 @@ syncprov_op_search( Operation *op, SlapReply *rs )
                        }
                        goto shortcut;
                }
+               /* Do we have a sessionlog for this search? */
+               sl=si->si_logs;
+               if ( sl ) {
+                       ldap_pvt_thread_mutex_lock( &sl->sl_mutex );
+                       if ( ber_bvcmp( &srs->sr_state.ctxcsn, &sl->sl_mincsn ) >= 0 ) {
+                               do_present = 0;
+                               /* mutex is unlocked in playlog */
+                               syncprov_playlog( op, rs, sl, &srs->sr_state.ctxcsn, &ctxcsn );
+                       } else {
+                               ldap_pvt_thread_mutex_unlock( &sl->sl_mutex );
+                       }
+               }
                /* Is the CSN still present in the database? */
                if ( syncprov_findcsn( op, FIND_CSN ) != LDAP_SUCCESS ) {
                        /* No, so a reload is required */
@@ -1545,14 +1851,16 @@ syncprov_op_search( Operation *op, SlapReply *rs )
 #endif
                } else {
                        gotstate = 1;
-                       /* If context has changed, check for Present UUIDs */
-                       if ( syncprov_findcsn( op, FIND_PRESENT ) != LDAP_SUCCESS ) {
+                       /* If changed and doing Present lookup, send Present UUIDs */
+                       if ( do_present && syncprov_findcsn( op, FIND_PRESENT ) !=
+                               LDAP_SUCCESS ) {
                                send_ldap_result( op, rs );
                                return rs->sr_err;
                        }
                }
        }
 
+shortcut:
        /* Append CSN range to search filter, save original filter
         * for persistent search evaluation
         */
@@ -1567,9 +1875,10 @@ syncprov_op_search( Operation *op, SlapReply *rs )
        fava->f_choice = LDAP_FILTER_LE;
        fava->f_ava = op->o_tmpalloc( sizeof(AttributeAssertion), op->o_tmpmemctx );
        fava->f_ava->aa_desc = slap_schema.si_ad_entryCSN;
-       ldap_pvt_thread_mutex_lock( &si->si_csn_mutex );
-       ber_dupbv_x( &fava->f_ava->aa_value, &si->si_ctxcsn, op->o_tmpmemctx );
-       ldap_pvt_thread_mutex_unlock( &si->si_csn_mutex );
+#ifdef LDAP_COMP_MATCH
+       fava->f_ava->aa_cf = NULL;
+#endif
+       ber_dupbv_x( &fava->f_ava->aa_value, &ctxcsn, op->o_tmpmemctx );
        fand->f_and = fava;
        if ( gotstate ) {
                fava->f_next = op->o_tmpalloc( sizeof(Filter), op->o_tmpmemctx );
@@ -1577,18 +1886,21 @@ syncprov_op_search( Operation *op, SlapReply *rs )
                fava->f_choice = LDAP_FILTER_GE;
                fava->f_ava = op->o_tmpalloc( sizeof(AttributeAssertion), op->o_tmpmemctx );
                fava->f_ava->aa_desc = slap_schema.si_ad_entryCSN;
-               ber_dupbv_x( &fava->f_ava->aa_value, srs->sr_state.ctxcsn, op->o_tmpmemctx );
+#ifdef LDAP_COMP_MATCH
+               fava->f_ava->aa_cf = NULL;
+#endif
+               ber_dupbv_x( &fava->f_ava->aa_value, &srs->sr_state.ctxcsn, op->o_tmpmemctx );
        }
        fava->f_next = op->ors_filter;
        op->ors_filter = fand;
        filter2bv_x( op, op->ors_filter, &op->ors_filterstr );
 
-shortcut:
        /* Let our callback add needed info to returned entries */
        cb = op->o_tmpcalloc(1, sizeof(slap_callback)+sizeof(searchstate), op->o_tmpmemctx);
        ss = (searchstate *)(cb+1);
        ss->ss_on = on;
        ss->ss_so = sop;
+       ss->ss_present = do_present;
        cb->sc_response = syncprov_search_response;
        cb->sc_cleanup = syncprov_search_cleanup;
        cb->sc_private = ss;
@@ -1679,6 +1991,33 @@ syncprov_db_config(
                si->si_chktime = atoi( argv[2] ) * 60;
                return 0;
 
+       } else if ( strcasecmp( argv[0], "syncprov-sessionlog" ) == 0 ) {
+               sessionlog *sl;
+               int size;
+               if ( argc != 2 ) {
+                       fprintf( stderr, "%s: line %d: wrong number of arguments in "
+                               "\"syncprov-sessionlog <size>\"\n", fname, lineno );
+                       return -1;
+               }
+               size = atoi( argv[1] );
+               if ( size < 0 ) {
+                       fprintf( stderr,
+                               "%s: line %d: session log size %d is negative\n",
+                               fname, lineno, size );
+                       return -1;
+               }
+               sl = si->si_logs;
+               if ( !sl ) {
+                       sl = ch_malloc( sizeof( sessionlog ) + LDAP_LUTIL_CSNSTR_BUFSIZE );
+                       sl->sl_mincsn.bv_val = (char *)(sl+1);
+                       sl->sl_mincsn.bv_len = 0;
+                       sl->sl_num = 0;
+                       sl->sl_head = sl->sl_tail = NULL;
+                       ldap_pvt_thread_mutex_init( &sl->sl_mutex );
+                       si->si_logs = sl;
+               }
+               sl->sl_size = size;
+               return 0;
        }
 
        return SLAP_CONF_UNKNOWN;
@@ -1717,6 +2056,15 @@ syncprov_db_open(
        Attribute *a;
        int rc;
 
+       if ( slapMode & SLAP_TOOL_MODE ) {
+               return 0;
+       }
+
+       rc = overlay_register_control( be, LDAP_CONTROL_SYNC );
+       if ( rc ) {
+               return rc;
+       }
+
        connection_fake_init( &conn, op, thrctx );
        op->o_bd = be;
        op->o_dn = be->be_rootdn;
@@ -1739,12 +2087,18 @@ syncprov_db_open(
                        si->si_ctxcsnbuf[si->si_ctxcsn.bv_len] = '\0';
                        strcpy( ctxcsnbuf, si->si_ctxcsnbuf );
                }
-               be_entry_release_r( op, e );
+               be_entry_release_rw( op, e, 0 );
                op->o_bd->bd_info = (BackendInfo *)on;
                op->o_req_dn = be->be_suffix[0];
                op->o_req_ndn = be->be_nsuffix[0];
                op->ors_scope = LDAP_SCOPE_SUBTREE;
                syncprov_findcsn( op, FIND_MAXCSN );
+       } else if ( SLAP_SYNC_SHADOW( op->o_bd )) {
+               /* If we're also a consumer, and we didn't find the context entry,
+                * then don't generate anything, wait for our provider to send it
+                * to us.
+                */
+               goto out;
        }
 
        if ( BER_BVISEMPTY( &si->si_ctxcsn ) ) {
@@ -1760,6 +2114,7 @@ syncprov_db_open(
                syncprov_checkpoint( op, &rs, on );
        }
 
+out:
        op->o_bd->bd_info = (BackendInfo *)on;
        return 0;
 }
@@ -1775,6 +2130,9 @@ syncprov_db_close(
     syncprov_info_t *si = (syncprov_info_t *)on->on_bi.bi_private;
        int i;
 
+       if ( slapMode & SLAP_TOOL_MODE ) {
+               return 0;
+       }
        if ( si->si_numops ) {
                Connection conn;
                char opbuf[OPERATION_BUFFER_SIZE];
@@ -1922,7 +2280,7 @@ static int syncprov_parseCtrl (
        sr = op->o_tmpcalloc( 1, sizeof(struct sync_control), op->o_tmpmemctx );
        sr->sr_rhint = rhint;
        if (!BER_BVISNULL(&cookie)) {
-               ber_bvarray_add( &sr->sr_state.octet_str, &cookie );
+               ber_dupbv( &sr->sr_state.octet_str, &cookie );
                slap_parse_sync_cookie( &sr->sr_state );
        }