]> git.sur5r.net Git - openldap/blobdiff - servers/slapd/overlays/syncprov.c
fix syncprov_qtask race, test062 crashes
[openldap] / servers / slapd / overlays / syncprov.c
index 2ac8425017b4a9c01e1bf641e39af7853b1ccb55..1d960673452c27eb9b12c4130bb76d56f4f433cf 100644 (file)
@@ -2,7 +2,7 @@
 /* syncprov.c - syncrepl provider */
 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
  *
- * Copyright 2004-2014 The OpenLDAP Foundation.
+ * Copyright 2004-2017 The OpenLDAP Foundation.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -28,7 +28,9 @@
 #include "config.h"
 #include "ldap_rq.h"
 
+#ifdef LDAP_DEVEL
 #define        CHECK_CSN       1
+#endif
 
 /* A modify request on a particular entry */
 typedef struct modinst {
@@ -43,21 +45,33 @@ typedef struct modtarget {
        ldap_pvt_thread_mutex_t mt_mutex;
 } modtarget;
 
+/* All the info of a psearch result that's shared between
+ * multiple queues
+ */
+typedef struct resinfo {
+       struct syncres *ri_list;
+       Entry *ri_e;
+       struct berval ri_dn;
+       struct berval ri_ndn;
+       struct berval ri_uuid;
+       struct berval ri_csn;
+       struct berval ri_cookie;
+       char ri_isref;
+       ldap_pvt_thread_mutex_t ri_mutex;
+} resinfo;
+
 /* A queued result of a persistent search */
 typedef struct syncres {
-       struct syncres *s_next;
-       Entry *s_e;
-       struct berval s_dn;
-       struct berval s_ndn;
-       struct berval s_uuid;
-       struct berval s_csn;
+       struct syncres *s_next; /* list of results on this psearch queue */
+       struct syncres *s_rilist;       /* list of psearches using this result */
+       resinfo *s_info;
        char s_mode;
-       char s_isreference;
 } syncres;
 
 /* Record of a persistent search */
 typedef struct syncops {
        struct syncops *s_next;
+       struct syncprov_info_t *s_si;
        struct berval   s_base;         /* ndn of search base */
        ID              s_eid;          /* entryID of search base */
        Operation       *s_op;          /* search op */
@@ -75,6 +89,7 @@ typedef struct syncops {
        int             s_inuse;        /* reference count */
        struct syncres *s_res;
        struct syncres *s_restail;
+       void *s_pool_cookie;
        ldap_pvt_thread_mutex_t s_mutex;
 } syncops;
 
@@ -157,13 +172,9 @@ typedef struct opcookie {
        short osid;     /* sid of op csn */
        short rsid;     /* sid of relay */
        short sreference;       /* Is the entry a reference? */
+       syncres ssres;
 } opcookie;
 
-typedef struct mutexint {
-       ldap_pvt_thread_mutex_t mi_mutex;
-       int mi_int;
-} mutexint;
-
 typedef struct fbase_cookie {
        struct berval *fdn;     /* DN of a modified entry, for scope testing */
        syncops *fss;   /* persistent search we're testing against */
@@ -760,49 +771,59 @@ again:
        return rc;
 }
 
-/* Should find a place to cache these */
-static mutexint *get_mutexint()
-{
-       mutexint *mi = ch_malloc( sizeof( mutexint ));
-       ldap_pvt_thread_mutex_init( &mi->mi_mutex );
-       mi->mi_int = 1;
-       return mi;
-}
-
-static void inc_mutexint( mutexint *mi )
+static void free_resinfo( syncres *sr )
 {
-       ldap_pvt_thread_mutex_lock( &mi->mi_mutex );
-       mi->mi_int++;
-       ldap_pvt_thread_mutex_unlock( &mi->mi_mutex );
+       syncres **st;
+       int freeit = 0;
+       ldap_pvt_thread_mutex_lock( &sr->s_info->ri_mutex );
+       for (st = &sr->s_info->ri_list; *st; st = &(*st)->s_rilist) {
+               if (*st == sr) {
+                       *st = sr->s_rilist;
+                       break;
+               }
+       }
+       if ( !sr->s_info->ri_list )
+               freeit = 1;
+       ldap_pvt_thread_mutex_unlock( &sr->s_info->ri_mutex );
+       if ( freeit ) {
+               ldap_pvt_thread_mutex_destroy( &sr->s_info->ri_mutex );
+               if ( sr->s_info->ri_e )
+                       entry_free( sr->s_info->ri_e );
+               if ( !BER_BVISNULL( &sr->s_info->ri_cookie ))
+                       ch_free( sr->s_info->ri_cookie.bv_val );
+               ch_free( sr->s_info );
+       }
 }
 
-/* return resulting counter */
-static int dec_mutexint( mutexint *mi )
-{
-       int i;
-       ldap_pvt_thread_mutex_lock( &mi->mi_mutex );
-       i = --mi->mi_int;
-       ldap_pvt_thread_mutex_unlock( &mi->mi_mutex );
-       if ( !i ) {
-               ldap_pvt_thread_mutex_destroy( &mi->mi_mutex );
-               ch_free( mi );
-       }
-       return i;
-}
+#define FS_UNLINK      1
+#define FS_LOCK                2
 
 static int
-syncprov_free_syncop( syncops *so )
+syncprov_free_syncop( syncops *so, int flags )
 {
        syncres *sr, *srnext;
        GroupAssertion *ga, *gnext;
 
-       ldap_pvt_thread_mutex_lock( &so->s_mutex );
+       if ( flags & FS_LOCK )
+               ldap_pvt_thread_mutex_lock( &so->s_mutex );
        /* already being freed, or still in use */
        if ( !so->s_inuse || --so->s_inuse > 0 ) {
-               ldap_pvt_thread_mutex_unlock( &so->s_mutex );
+               if ( flags & FS_LOCK )
+                       ldap_pvt_thread_mutex_unlock( &so->s_mutex );
                return 0;
        }
        ldap_pvt_thread_mutex_unlock( &so->s_mutex );
+       if (( flags & FS_UNLINK ) && so->s_si ) {
+               syncops **sop;
+               ldap_pvt_thread_mutex_lock( &so->s_si->si_ops_mutex );
+               for ( sop = &so->s_si->si_ops; *sop; sop = &(*sop)->s_next ) {
+                       if ( *sop == so ) {
+                               *sop = so->s_next;
+                               break;
+                       }
+               }
+               ldap_pvt_thread_mutex_unlock( &so->s_si->si_ops_mutex );
+       }
        if ( so->s_flags & PS_IS_DETACHED ) {
                filter_free( so->s_op->ors_filter );
                for ( ga = so->s_op->o_groups; ga; ga=gnext ) {
@@ -814,12 +835,7 @@ syncprov_free_syncop( syncops *so )
        ch_free( so->s_base.bv_val );
        for ( sr=so->s_res; sr; sr=srnext ) {
                srnext = sr->s_next;
-               if ( sr->s_e ) {
-                       if ( !dec_mutexint( sr->s_e->e_private )) {
-                               sr->s_e->e_private = NULL;
-                               entry_free( sr->s_e );
-                       }
-               }
+               free_resinfo( sr );
                ch_free( sr );
        }
        ldap_pvt_thread_mutex_destroy( &so->s_mutex );
@@ -829,7 +845,7 @@ syncprov_free_syncop( syncops *so )
 
 /* Send a persistent search response */
 static int
-syncprov_sendresp( Operation *op, opcookie *opc, syncops *so, int mode )
+syncprov_sendresp( Operation *op, resinfo *ri, syncops *so, int mode )
 {
        SlapReply rs = { REP_SEARCH };
        struct berval cookie, csns[2];
@@ -842,7 +858,7 @@ syncprov_sendresp( Operation *op, opcookie *opc, syncops *so, int mode )
        rs.sr_ctrls = op->o_tmpalloc( sizeof(LDAPControl *)*2, op->o_tmpmemctx );
        rs.sr_ctrls[1] = NULL;
        rs.sr_flags = REP_CTRLS_MUSTBEFREED;
-       csns[0] = opc->sctxcsn;
+       csns[0] = ri->ri_csn;
        BER_BVZERO( &csns[1] );
        slap_compose_sync_cookie( op, &cookie, csns, so->s_rid, slap_serverID ? slap_serverID : -1 );
 
@@ -858,20 +874,20 @@ syncprov_sendresp( Operation *op, opcookie *opc, syncops *so, int mode )
 
        e_uuid.e_attrs = &a_uuid;
        a_uuid.a_desc = slap_schema.si_ad_entryUUID;
-       a_uuid.a_nvals = &opc->suuid;
+       a_uuid.a_nvals = &ri->ri_uuid;
        rs.sr_err = syncprov_state_ctrl( op, &rs, &e_uuid,
                mode, rs.sr_ctrls, 0, 1, &cookie );
        op->o_tmpfree( cookie.bv_val, op->o_tmpmemctx );
 
        rs.sr_entry = &e_uuid;
        if ( mode == LDAP_SYNC_ADD || mode == LDAP_SYNC_MODIFY ) {
-               e_uuid = *opc->se;
+               e_uuid = *ri->ri_e;
                e_uuid.e_private = NULL;
        }
 
        switch( mode ) {
        case LDAP_SYNC_ADD:
-               if ( opc->sreference && so->s_op->o_managedsait <= SLAP_CONTROL_IGNORED ) {
+               if ( ri->ri_isref && so->s_op->o_managedsait <= SLAP_CONTROL_IGNORED ) {
                        rs.sr_ref = get_entry_referrals( op, rs.sr_entry );
                        rs.sr_err = send_search_reference( op, &rs );
                        ber_bvarray_free( rs.sr_ref );
@@ -884,9 +900,9 @@ syncprov_sendresp( Operation *op, opcookie *opc, syncops *so, int mode )
                break;
        case LDAP_SYNC_DELETE:
                e_uuid.e_attrs = NULL;
-               e_uuid.e_name = opc->sdn;
-               e_uuid.e_nname = opc->sndn;
-               if ( opc->sreference && so->s_op->o_managedsait <= SLAP_CONTROL_IGNORED ) {
+               e_uuid.e_name = ri->ri_dn;
+               e_uuid.e_nname = ri->ri_ndn;
+               if ( ri->ri_isref && so->s_op->o_managedsait <= SLAP_CONTROL_IGNORED ) {
                        struct berval bv = BER_BVNULL;
                        rs.sr_ref = &bv;
                        rs.sr_err = send_search_reference( op, &rs );
@@ -907,13 +923,9 @@ syncprov_qstart( syncops *so );
 static int
 syncprov_qplay( Operation *op, syncops *so )
 {
-       slap_overinst *on = LDAP_SLIST_FIRST(&so->s_op->o_extra)->oe_key;
        syncres *sr;
-       opcookie opc;
        int rc = 0;
 
-       opc.son = on;
-
        do {
                ldap_pvt_thread_mutex_lock( &so->s_mutex );
                sr = so->s_res;
@@ -931,25 +943,13 @@ syncprov_qplay( Operation *op, syncops *so )
                                SlapReply rs = { REP_INTERMEDIATE };
 
                                rc = syncprov_sendinfo( op, &rs, LDAP_TAG_SYNC_NEW_COOKIE,
-                                       &sr->s_csn, 0, NULL, 0 );
+                                       &sr->s_info->ri_cookie, 0, NULL, 0 );
                        } else {
-                               opc.sdn = sr->s_dn;
-                               opc.sndn = sr->s_ndn;
-                               opc.suuid = sr->s_uuid;
-                               opc.sctxcsn = sr->s_csn;
-                               opc.sreference = sr->s_isreference;
-                               opc.se = sr->s_e;
-
-                               rc = syncprov_sendresp( op, &opc, so, sr->s_mode );
+                               rc = syncprov_sendresp( op, sr->s_info, so, sr->s_mode );
                        }
                }
 
-               if ( sr->s_e ) {
-                       if ( !dec_mutexint( sr->s_e->e_private )) {
-                               sr->s_e->e_private = NULL;
-                               entry_free ( sr->s_e );
-                       }
-               }
+               free_resinfo( sr );
                ch_free( sr );
 
                if ( so->s_op->o_abandon )
@@ -968,11 +968,8 @@ syncprov_qplay( Operation *op, syncops *so )
 
        if ( rc == 0 && so->s_res ) {
                syncprov_qstart( so );
-       } else {
-               so->s_flags ^= PS_TASK_QUEUED;
        }
 
-       ldap_pvt_thread_mutex_unlock( &so->s_mutex );
        return rc;
 }
 
@@ -1008,8 +1005,17 @@ syncprov_qtask( void *ctx, void *arg )
 
        rc = syncprov_qplay( op, so );
 
+       /* if an error occurred, or no responses left, task is no longer queued */
+       if ( !rc && !so->s_res )
+               rc = 1;
+
        /* decrement use count... */
-       syncprov_free_syncop( so );
+       if ( !syncprov_free_syncop( so, FS_UNLINK )) {
+               if ( rc )
+                       /* if we didn't unlink, and task is no longer queued, clear flag */
+                       so->s_flags ^= PS_TASK_QUEUED;
+               ldap_pvt_thread_mutex_unlock( &so->s_mutex );
+       }
 
        return NULL;
 }
@@ -1020,8 +1026,8 @@ syncprov_qstart( syncops *so )
 {
        so->s_flags |= PS_TASK_QUEUED;
        so->s_inuse++;
-       ldap_pvt_thread_pool_submit( &connection_pool, 
-               syncprov_qtask, so );
+       ldap_pvt_thread_pool_submit2( &connection_pool,
+               syncprov_qtask, so, &so->s_pool_cookie );
 }
 
 /* Queue a persistent search response */
@@ -1029,48 +1035,79 @@ static int
 syncprov_qresp( opcookie *opc, syncops *so, int mode )
 {
        syncres *sr;
+       resinfo *ri;
        int srsize;
-       struct berval cookie = opc->sctxcsn;
-
-       if ( mode == LDAP_SYNC_NEW_COOKIE ) {
-               syncprov_info_t *si = opc->son->on_bi.bi_private;
-
-               slap_compose_sync_cookie( NULL, &cookie, si->si_ctxcsn,
-                       so->s_rid, slap_serverID ? slap_serverID : -1);
-       }
+       struct berval csn = opc->sctxcsn;
 
-       srsize = sizeof(syncres) + opc->suuid.bv_len + 1 +
-               opc->sdn.bv_len + 1 + opc->sndn.bv_len + 1;
-       if ( cookie.bv_len )
-               srsize += cookie.bv_len + 1;
-       sr = ch_malloc( srsize );
+       sr = ch_malloc( sizeof( syncres ));
        sr->s_next = NULL;
-       sr->s_e = opc->se;
-       /* bump refcount on this entry */
-       if ( opc->se )
-               inc_mutexint( opc->se->e_private );
-       sr->s_dn.bv_val = (char *)(sr + 1);
-       sr->s_dn.bv_len = opc->sdn.bv_len;
        sr->s_mode = mode;
-       sr->s_isreference = opc->sreference;
-       sr->s_ndn.bv_val = lutil_strcopy( sr->s_dn.bv_val,
-                opc->sdn.bv_val ) + 1;
-       sr->s_ndn.bv_len = opc->sndn.bv_len;
-       sr->s_uuid.bv_val = lutil_strcopy( sr->s_ndn.bv_val,
-                opc->sndn.bv_val ) + 1;
-       sr->s_uuid.bv_len = opc->suuid.bv_len;
-       AC_MEMCPY( sr->s_uuid.bv_val, opc->suuid.bv_val, opc->suuid.bv_len );
-       if ( cookie.bv_len ) {
-               sr->s_csn.bv_val = sr->s_uuid.bv_val + sr->s_uuid.bv_len + 1;
-               strcpy( sr->s_csn.bv_val, cookie.bv_val );
-       } else {
-               sr->s_csn.bv_val = NULL;
-       }
-       sr->s_csn.bv_len = cookie.bv_len;
+       if ( !opc->ssres.s_info ) {
+
+               srsize = sizeof( resinfo );
+               if ( csn.bv_len )
+                       srsize += csn.bv_len + 1;
+
+               if ( opc->se ) {
+                       Attribute *a;
+                       ri = ch_malloc( srsize );
+                       ri->ri_dn = opc->se->e_name;
+                       ri->ri_ndn = opc->se->e_nname;
+                       a = attr_find( opc->se->e_attrs, slap_schema.si_ad_entryUUID );
+                       if ( a )
+                               ri->ri_uuid = a->a_nvals[0];
+                       else
+                               ri->ri_uuid.bv_len = 0;
+                       if ( csn.bv_len ) {
+                               ri->ri_csn.bv_val = (char *)(ri + 1);
+                               ri->ri_csn.bv_len = csn.bv_len;
+                               memcpy( ri->ri_csn.bv_val, csn.bv_val, csn.bv_len );
+                               ri->ri_csn.bv_val[csn.bv_len] = '\0';
+                       } else {
+                               ri->ri_csn.bv_val = NULL;
+                       }
+               } else {
+                       srsize += opc->suuid.bv_len +
+                               opc->sdn.bv_len + 1 + opc->sndn.bv_len + 1;
+                       ri = ch_malloc( srsize );
+                       ri->ri_dn.bv_val = (char *)(ri + 1);
+                       ri->ri_dn.bv_len = opc->sdn.bv_len;
+                       ri->ri_ndn.bv_val = lutil_strcopy( ri->ri_dn.bv_val,
+                               opc->sdn.bv_val ) + 1;
+                       ri->ri_ndn.bv_len = opc->sndn.bv_len;
+                       ri->ri_uuid.bv_val = lutil_strcopy( ri->ri_ndn.bv_val,
+                               opc->sndn.bv_val ) + 1;
+                       ri->ri_uuid.bv_len = opc->suuid.bv_len;
+                       AC_MEMCPY( ri->ri_uuid.bv_val, opc->suuid.bv_val, opc->suuid.bv_len );
+                       if ( csn.bv_len ) {
+                               ri->ri_csn.bv_val = ri->ri_uuid.bv_val + ri->ri_uuid.bv_len;
+                               memcpy( ri->ri_csn.bv_val, csn.bv_val, csn.bv_len );
+                               ri->ri_csn.bv_val[csn.bv_len] = '\0';
+                       } else {
+                               ri->ri_csn.bv_val = NULL;
+                       }
+               }
+               ri->ri_list = &opc->ssres;
+               ri->ri_e = opc->se;
+               ri->ri_csn.bv_len = csn.bv_len;
+               ri->ri_isref = opc->sreference;
+               BER_BVZERO( &ri->ri_cookie );
+               ldap_pvt_thread_mutex_init( &ri->ri_mutex );
+               opc->se = NULL;
+               opc->ssres.s_info = ri;
+       }
+       ri = opc->ssres.s_info;
+       sr->s_info = ri;
+       ldap_pvt_thread_mutex_lock( &ri->ri_mutex );
+       sr->s_rilist = ri->ri_list;
+       ri->ri_list = sr;
+       if ( mode == LDAP_SYNC_NEW_COOKIE && BER_BVISNULL( &ri->ri_cookie )) {
+               syncprov_info_t *si = opc->son->on_bi.bi_private;
 
-       if ( mode == LDAP_SYNC_NEW_COOKIE && cookie.bv_val ) {
-               ch_free( cookie.bv_val );
+               slap_compose_sync_cookie( NULL, &ri->ri_cookie, si->si_ctxcsn,
+                       so->s_rid, slap_serverID ? slap_serverID : -1);
        }
+       ldap_pvt_thread_mutex_unlock( &ri->ri_mutex );
 
        ldap_pvt_thread_mutex_lock( &so->s_mutex );
        if ( !so->s_res ) {
@@ -1105,7 +1142,7 @@ syncprov_drop_psearch( syncops *so, int lock )
                if ( lock )
                        ldap_pvt_thread_mutex_unlock( &so->s_op->o_conn->c_mutex );
        }
-       return syncprov_free_syncop( so );
+       return syncprov_free_syncop( so, FS_LOCK );
 }
 
 static int
@@ -1123,15 +1160,14 @@ syncprov_op_abandon( Operation *op, SlapReply *rs )
 {
        slap_overinst           *on = (slap_overinst *)op->o_bd->bd_info;
        syncprov_info_t         *si = on->on_bi.bi_private;
-       syncops *so, *soprev;
+       syncops *so, **sop;
 
        ldap_pvt_thread_mutex_lock( &si->si_ops_mutex );
-       for ( so=si->si_ops, soprev = (syncops *)&si->si_ops; so;
-               soprev=so, so=so->s_next ) {
+       for ( sop=&si->si_ops; (so = *sop); sop = &(*sop)->s_next ) {
                if ( so->s_op->o_connid == op->o_connid &&
                        so->s_op->o_msgid == op->orn_msgid ) {
                                so->s_op->o_abandon = 1;
-                               soprev->s_next = so->s_next;
+                               *sop = so->s_next;
                                break;
                }
        }
@@ -1191,10 +1227,8 @@ syncprov_matchops( Operation *op, opcookie *opc, int saveit )
                rc = overlay_entry_get_ov( op, fc.fdn, NULL, NULL, 0, &e, on );
                /* If we're sending responses now, make a copy and unlock the DB */
                if ( e && !saveit ) {
-                       if ( !opc->se ) {
+                       if ( !opc->se )
                                opc->se = entry_dup( e );
-                               opc->se->e_private = get_mutexint();
-                       }
                        overlay_entry_release_ov( op, e, 0, on );
                        e = opc->se;
                }
@@ -1205,10 +1239,8 @@ syncprov_matchops( Operation *op, opcookie *opc, int saveit )
        } else {
                e = op->ora_e;
                if ( !saveit ) {
-                       if ( !opc->se ) {
+                       if ( !opc->se )
                                opc->se = entry_dup( e );
-                               opc->se->e_private = get_mutexint();
-                       }
                        e = opc->se;
                }
        }
@@ -1343,7 +1375,7 @@ syncprov_matchops( Operation *op, opcookie *opc, int saveit )
                         * with saveit == TRUE
                         */
                        snext = ss->s_next;
-                       if ( syncprov_free_syncop( ss ) ) {
+                       if ( syncprov_free_syncop( ss, FS_LOCK ) ) {
                                *pss = snext;
                                gonext = 0;
                        }
@@ -1359,12 +1391,11 @@ syncprov_matchops( Operation *op, opcookie *opc, int saveit )
                        overlay_entry_release_ov( op, e, 0, on );
                op->o_bd = b0;
        }
-       if ( opc->se && !saveit ) {
-               if ( !dec_mutexint( opc->se->e_private )) {
-                       opc->se->e_private = NULL;
+       if ( !saveit ) {
+               if ( opc->ssres.s_info )
+                       free_resinfo( &opc->ssres );
+               else if ( opc->se )
                        entry_free( opc->se );
-                       opc->se = NULL;
-               }
        }
        if ( freefdn ) {
                op->o_tmpfree( fc.fdn->bv_val, op->o_tmpmemctx );
@@ -1389,15 +1420,23 @@ syncprov_op_cleanup( Operation *op, SlapReply *rs )
 
        for (sm = opc->smatches; sm; sm=snext) {
                snext = sm->sm_next;
-               syncprov_free_syncop( sm->sm_op );
+               syncprov_free_syncop( sm->sm_op, FS_LOCK|FS_UNLINK );
                op->o_tmpfree( sm, op->o_tmpmemctx );
        }
 
        /* Remove op from lock table */
        mt = opc->smt;
        if ( mt ) {
+               modinst *mi = (modinst *)(opc+1), **m2;
                ldap_pvt_thread_mutex_lock( &mt->mt_mutex );
-               mt->mt_mods = mt->mt_mods->mi_next;
+               for (m2 = &mt->mt_mods; ; m2 = &(*m2)->mi_next) {
+                       if ( *m2 == mi ) {
+                               *m2 = mi->mi_next;
+                               if ( mt->mt_tail == mi )
+                                       mt->mt_tail = ( m2 == &mt->mt_mods ) ? NULL : (modinst *)m2;
+                               break;
+                       }
+               }
                /* If there are more, promote the next one */
                if ( mt->mt_mods ) {
                        ldap_pvt_thread_mutex_unlock( &mt->mt_mutex );
@@ -1466,6 +1505,7 @@ syncprov_checkpoint( Operation *op, slap_overinst *on )
        opm.o_bd->bd_info = on->on_info->oi_orig;
        opm.o_managedsait = SLAP_CONTROL_NONCRITICAL;
        opm.o_no_schema_check = 1;
+       opm.o_opid = -1;
        opm.o_bd->be_modify( &opm, &rsm );
 
        if ( rsm.sr_err == LDAP_NO_SUCH_OBJECT &&
@@ -1522,7 +1562,7 @@ syncprov_add_slog( Operation *op )
                }
 
                /* Allocate a record. UUIDs are not NUL-terminated. */
-               se = ch_malloc( sizeof( slog_entry ) + opc->suuid.bv_len + 
+               se = ch_malloc( sizeof( slog_entry ) + opc->suuid.bv_len +
                        op->o_csn.bv_len + 1 );
                se->se_next = NULL;
                se->se_tag = op->o_tag;
@@ -1568,7 +1608,7 @@ syncprov_add_slog( Operation *op )
                }
                sl->sl_num++;
                while ( sl->sl_num > sl->sl_size ) {
-                       int i, j;
+                       int i;
                        se = sl->sl_head;
                        sl->sl_head = se->se_next;
                        for ( i=0; i<sl->sl_numcsns; i++ )
@@ -1889,10 +1929,13 @@ syncprov_op_response( Operation *op, SlapReply *rs )
                        } else {
                        ldap_pvt_thread_rdwr_wunlock( &si->si_csn_rwlock );
                        }
+                       if ( csn_changed )
+                               si->si_numops++;
                        goto leave;
                }
 
-               si->si_numops++;
+               if ( csn_changed )
+                       si->si_numops++;
                if ( si->si_chkops || si->si_chktime ) {
                        /* Never checkpoint adding the context entry,
                         * it will deadlock
@@ -1949,6 +1992,8 @@ syncprov_op_response( Operation *op, SlapReply *rs )
                                                continue;
                                        syncprov_qresp( opc, sm->sm_op, LDAP_SYNC_DELETE );
                                }
+                               if ( opc->ssres.s_info )
+                                       free_resinfo( &opc->ssres );
                                break;
                        }
                }
@@ -2084,24 +2129,38 @@ syncprov_op_mod( Operation *op, SlapReply *rs )
 
                /* See if we're already modifying this entry... */
                mtdummy.mt_dn = op->o_req_ndn;
+retry:
                ldap_pvt_thread_mutex_lock( &si->si_mods_mutex );
                mt = avl_find( si->si_mods, &mtdummy, sp_avl_cmp );
                if ( mt ) {
                        ldap_pvt_thread_mutex_lock( &mt->mt_mutex );
                        if ( mt->mt_mods == NULL ) {
                                /* Cannot reuse this mt, as another thread is about
-                                * to release it in syncprov_op_cleanup.
+                                * to release it in syncprov_op_cleanup. Wait for them
+                                * to finish; our own insert is required to succeed.
                                 */
                                ldap_pvt_thread_mutex_unlock( &mt->mt_mutex );
-                               mt = NULL;
+                               ldap_pvt_thread_mutex_unlock( &si->si_mods_mutex );
+                               ldap_pvt_thread_yield();
+                               goto retry;
                        }
                }
                if ( mt ) {
-                       ldap_pvt_thread_mutex_unlock( &si->si_mods_mutex );
                        mt->mt_tail->mi_next = mi;
                        mt->mt_tail = mi;
+                       ldap_pvt_thread_mutex_unlock( &si->si_mods_mutex );
                        /* wait for this op to get to head of list */
                        while ( mt->mt_mods != mi ) {
+                               modinst *m2;
+                               /* don't wait on other mods from the same thread */
+                               for ( m2 = mt->mt_mods; m2; m2 = m2->mi_next ) {
+                                       if ( m2->mi_op->o_threadctx == op->o_threadctx ) {
+                                               break;
+                                       }
+                               }
+                               if ( m2 )
+                                       break;
+
                                ldap_pvt_thread_mutex_unlock( &mt->mt_mutex );
                                /* FIXME: if dynamic config can delete overlays or
                                 * databases we'll have to check for cleanup here.
@@ -2117,12 +2176,21 @@ syncprov_op_mod( Operation *op, SlapReply *rs )
 
                                /* clean up if the caller is giving up */
                                if ( op->o_abandon ) {
-                                       modinst *m2;
-                                       for ( m2 = mt->mt_mods; m2 && m2->mi_next != mi;
-                                               m2 = m2->mi_next );
-                                       if ( m2 ) {
-                                               m2->mi_next = mi->mi_next;
-                                               if ( mt->mt_tail == mi ) mt->mt_tail = m2;
+                                       modinst **m2;
+                                       slap_callback **sc;
+                                       for (m2 = &mt->mt_mods; ; m2 = &(*m2)->mi_next) {
+                                               if ( *m2 == mi ) {
+                                                       *m2 = mi->mi_next;
+                                                       if ( mt->mt_tail == mi )
+                                                               mt->mt_tail = ( m2 == &mt->mt_mods ) ? NULL : (modinst *)m2;
+                                                       break;
+                                               }
+                                       }
+                                       for (sc = &op->o_callback; ; sc = &(*sc)->sc_next) {
+                                               if ( *sc == cb ) {
+                                                       *sc = cb->sc_next;
+                                                       break;
+                                               }
                                        }
                                        op->o_tmpfree( cb, op->o_tmpmemctx );
                                        ldap_pvt_thread_mutex_unlock( &mt->mt_mutex );
@@ -2380,7 +2448,7 @@ syncprov_search_response( Operation *op, SlapReply *rs )
                } else {
                /* It's RefreshAndPersist, transition to Persist phase */
                        syncprov_sendinfo( op, rs, ( ss->ss_flags & SS_PRESENT ) ?
-                               LDAP_TAG_SYNC_REFRESH_PRESENT : LDAP_TAG_SYNC_REFRESH_DELETE,
+                               LDAP_TAG_SYNC_REFRESH_PRESENT : LDAP_TAG_SYNC_REFRESH_DELETE,
                                ( ss->ss_flags & SS_CHANGED ) ? &cookie : NULL,
                                1, NULL, 0 );
                        if ( !BER_BVISNULL( &cookie ))
@@ -2447,7 +2515,7 @@ syncprov_op_search( Operation *op, SlapReply *rs )
                syncops so = {0};
                fbase_cookie fc;
                opcookie opc;
-               slap_callback sc;
+               slap_callback sc = {0};
 
                fc.fss = &so;
                fc.fbase = 0;
@@ -2470,7 +2538,6 @@ syncprov_op_search( Operation *op, SlapReply *rs )
                }
                sop = ch_malloc( sizeof( syncops ));
                *sop = so;
-               ldap_pvt_thread_mutex_init( &sop->s_mutex );
                sop->s_rid = srs->sr_state.rid;
                sop->s_sid = srs->sr_state.sid;
                /* set refcount=2 to prevent being freed out from under us
@@ -2495,12 +2562,21 @@ syncprov_op_search( Operation *op, SlapReply *rs )
                                ldap_pvt_thread_yield();
                        ldap_pvt_thread_mutex_lock( &si->si_ops_mutex );
                }
+               if ( op->o_abandon ) {
+                       ldap_pvt_thread_mutex_unlock( &si->si_ops_mutex );
+                       ch_free( sop );
+                       return SLAPD_ABANDON;
+               }
+               ldap_pvt_thread_mutex_init( &sop->s_mutex );
                sop->s_next = si->si_ops;
+               sop->s_si = si;
                si->si_ops = sop;
                ldap_pvt_thread_mutex_unlock( &si->si_ops_mutex );
        }
 
-       /* snapshot the ctxcsn */
+       /* snapshot the ctxcsn
+        * Note: this must not be done before the psearch setup. (ITS#8365)
+        */
        ldap_pvt_thread_rdwr_rlock( &si->si_csn_rwlock );
        numcsns = si->si_numcsns;
        if ( numcsns ) {
@@ -2514,17 +2590,19 @@ syncprov_op_search( Operation *op, SlapReply *rs )
        }
        dirty = si->si_dirty;
        ldap_pvt_thread_rdwr_runlock( &si->si_csn_rwlock );
-       
+
        /* If we have a cookie, handle the PRESENT lookups */
        if ( srs->sr_state.ctxcsn ) {
                sessionlog *sl;
                int i, j;
 
-               /* If we don't have any CSN of our own yet, pretend nothing
-                * has changed.
+               /* If we don't have any CSN of our own yet, bail out.
                 */
-               if ( !numcsns )
-                       goto no_change;
+               if ( !numcsns ) {
+                       rs->sr_err = LDAP_UNWILLING_TO_PERFORM;
+                       rs->sr_text = "consumer has state info but provider doesn't!";
+                       goto bailout;
+               }
 
                if ( !si->si_nopres )
                        do_present = SS_PRESENT;
@@ -2590,6 +2668,10 @@ syncprov_op_search( Operation *op, SlapReply *rs )
                        /* our state is older, complain to consumer */
                                rs->sr_err = LDAP_UNWILLING_TO_PERFORM;
                                rs->sr_text = "consumer state is newer than provider!";
+                               Log4( LDAP_DEBUG_SYNC, ldap_syslog_level,
+                                       "consumer %d state %s is newer than provider %d state %s\n",
+                                       sids[i], srs->sr_state.ctxcsn[i].bv_val, sids[j], /* == slap_serverID */
+                                       ctxcsn[j].bv_val);
 bailout:
                                if ( sop ) {
                                        syncops **sp = &si->si_ops;
@@ -2605,7 +2687,7 @@ bailout:
                                send_ldap_result( op, rs );
                                return rs->sr_err;
                        }
-               }
+               }
                if ( BER_BVISEMPTY( &mincsn )) {
                        mincsn = maxcsn;
                        minsid = maxsid;
@@ -2614,7 +2696,7 @@ bailout:
                /* If nothing has changed, shortcut it */
                if ( !changed && !dirty ) {
                        do_present = 0;
-no_change:             if ( !(op->o_sync_mode & SLAP_SYNC_PERSIST) ) {
+no_change:     if ( !(op->o_sync_mode & SLAP_SYNC_PERSIST) ) {
                                LDAPControl     *ctrls[2];
 
                                ctrls[0] = NULL;
@@ -2701,6 +2783,9 @@ no_change:                if ( !(op->o_sync_mode & SLAP_SYNC_PERSIST) ) {
                        }
                }
        } else {
+               /* The consumer knows nothing, we know nothing. OK. */
+               if (!numcsns)
+                       goto no_change;
                /* No consumer state, assume something has changed */
                changed = SS_CHANGED;
        }
@@ -3105,7 +3190,7 @@ syncprov_db_open(
                char csnbuf[ LDAP_PVT_CSNSTR_BUFSIZE ];
                struct berval csn;
 
-               if ( SLAP_SYNC_SHADOW( op->o_bd )) {
+               if ( slap_serverID || SLAP_SYNC_SHADOW( op->o_bd )) {
                /* If we're also a consumer, then don't generate anything.
                 * Wait for our provider to send it to us, or for a local
                 * modify if we have multimaster.
@@ -3148,8 +3233,8 @@ syncprov_db_close(
        ConfigReply *cr
 )
 {
-    slap_overinst   *on = (slap_overinst *) be->bd_info;
-    syncprov_info_t *si = (syncprov_info_t *)on->on_bi.bi_private;
+       slap_overinst   *on = (slap_overinst *) be->bd_info;
+       syncprov_info_t *si = (syncprov_info_t *)on->on_bi.bi_private;
 #ifdef SLAP_CONFIG_DELETE
        syncops *so, *sonext;
 #endif /* SLAP_CONFIG_DELETE */
@@ -3180,7 +3265,10 @@ syncprov_db_close(
                        rs.sr_err = LDAP_UNAVAILABLE;
                        send_ldap_result( so->s_op, &rs );
                        sonext=so->s_next;
-                       syncprov_drop_psearch( so, 0);
+                       if ( so->s_flags & PS_TASK_QUEUED )
+                               ldap_pvt_thread_pool_retract( so->s_pool_cookie );
+                       if ( !syncprov_drop_psearch( so, 0 ))
+                               so->s_si = NULL;
                }
                si->si_ops=NULL;
                ldap_pvt_thread_mutex_unlock( &si->si_ops_mutex );
@@ -3188,7 +3276,7 @@ syncprov_db_close(
        overlay_unregister_control( be, LDAP_CONTROL_SYNC );
 #endif /* SLAP_CONFIG_DELETE */
 
-    return 0;
+       return 0;
 }
 
 static int