Use batched write txns in refresh

Experimental - write 500 updates per txn instead of 1:1

diff --git a/servers/slapd/back-mdb/back-mdb.h b/servers/slapd/back-mdb/back-mdb.h
index f07428fd32cdf0e3093662f8ca12f7f880b8afea..8fe2561b66c3201ece7e42a33d800f7366de1329 100644 (file)
--- a/servers/slapd/back-mdb/back-mdb.h
+++ b/servers/slapd/back-mdb/back-mdb.h
@@ -115,6 +115,7 @@ typedef struct mdb_op_info {
 } mdb_op_info;
 #define MOI_READER     0x01
 #define MOI_FREEIT     0x02
+#define MOI_KEEPER     0x04
 
 /* Copy an ID "src" to pointer "dst" in big-endian byte order */
 #define MDB_ID2DISK( src, dst )        \

diff --git a/servers/slapd/back-mdb/id2entry.c b/servers/slapd/back-mdb/id2entry.c
index 4792d6fff46d6c8f926be31a30316b41b50006ea..d080f3f93e6f49b08b1c8e42ae4bc8f849de8ffe 100644 (file)
--- a/servers/slapd/back-mdb/id2entry.c
+++ b/servers/slapd/back-mdb/id2entry.c
@@ -284,7 +284,7 @@ int mdb_entry_release(
                                mdb_entry_return( op, e );
                                moi = (mdb_op_info *)oex;
                                /* If it was setup by entry_get we should probably free it */
-                               if ( moi->moi_flag & MOI_FREEIT ) {
+                               if (( moi->moi_flag & (MOI_FREEIT|MOI_KEEPER)) == MOI_FREEIT ) {
                                        moi->moi_ref--;
                                        if ( moi->moi_ref < 1 ) {
                                                mdb_txn_reset( moi->moi_txn );
@@ -541,7 +541,12 @@ int mdb_txn( Operation *op, int txnop, OpExtra **ptr )
 
        switch( txnop ) {
        case SLAP_TXN_BEGIN:
-               return mdb_opinfo_get( op, mdb, 0, moip );
+               rc = mdb_opinfo_get( op, mdb, 0, moip );
+               if ( !rc ) {
+                       moi = *moip;
+                       moi->moi_flag |= MOI_KEEPER;
+               }
+               return rc;
        case SLAP_TXN_COMMIT:
                rc = mdb_txn_commit( moi->moi_txn );
                op->o_tmpfree( moi, op->o_tmpmemctx );

diff --git a/servers/slapd/syncrepl.c b/servers/slapd/syncrepl.c
index 255ca5bcc7deec260acfe581ce1ca75acccf504c..e01297b9bc8e8b219981f27f8e26e272aed91356 100644 (file)
--- a/servers/slapd/syncrepl.c
+++ b/servers/slapd/syncrepl.c
@@ -110,6 +110,10 @@ typedef struct syncinfo_s {
        int                     si_refreshDelete;
        int                     si_refreshPresent;
        int                     si_refreshDone;
+       int                     si_refreshCount;
+       time_t          si_refreshBeg;
+       time_t          si_refreshEnd;
+       OpExtra         *si_refreshTxn;
        int                     si_syncdata;
        int                     si_logstate;
        int                     si_lazyCommit;
@@ -736,6 +740,11 @@ do_syncrep1(
        }
 
        si->si_refreshDone = 0;
+       si->si_refreshBeg = slap_get_time();
+       si->si_refreshCount = 0;
+       si->si_refreshTxn = NULL;
+       Debug( LDAP_DEBUG_ANY, "do_syncrep1: %s starting refresh\n",
+               si->si_ridtxt, 0, 0 );
 
        rc = ldap_sync_search( si, op->o_tmpmemctx );
 
@@ -1267,6 +1276,15 @@ do_syncrep2(
                                        {
                                                si->si_refreshDone = 1;
                                        }
+                                       if ( si->si_refreshDone ) {
+                                               if ( si->si_refreshCount ) {
+                                                       LDAP_SLIST_REMOVE( &op->o_extra, si->si_refreshTxn, OpExtra, oe_next );
+                                                       op->o_bd->bd_info->bi_op_txn( op, SLAP_TXN_COMMIT, &si->si_refreshTxn );
+                                               }
+                                               si->si_refreshEnd = slap_get_time();
+       Debug( LDAP_DEBUG_ANY, "do_syncrep1: %s finished refresh\n",
+               si->si_ridtxt, 0, 0 );
+                                       }
                                        ber_scanf( ber, /*"{"*/ "}" );
                                        if ( abs(si->si_type) == LDAP_SYNC_REFRESH_AND_PERSIST &&
                                                si->si_refreshDone )
@@ -2933,8 +2951,20 @@ syncrepl_entry(
                slap_queue_csn( op, syncCSN );
        }
 
-       if ( !si->si_refreshDone && si->si_lazyCommit )
-               op->o_lazyCommit = SLAP_CONTROL_NONCRITICAL;
+       if ( !si->si_refreshDone ) {
+               if ( si->si_lazyCommit )
+                       op->o_lazyCommit = SLAP_CONTROL_NONCRITICAL;
+               if ( si->si_refreshCount == 500 ) {
+                       LDAP_SLIST_REMOVE( &op->o_extra, si->si_refreshTxn, OpExtra, oe_next );
+                       op->o_bd->bd_info->bi_op_txn( op, SLAP_TXN_COMMIT, &si->si_refreshTxn );
+                       si->si_refreshCount = 0;
+                       si->si_refreshTxn = NULL;
+               }
+               if ( !si->si_refreshCount ) {
+                       op->o_bd->bd_info->bi_op_txn( op, SLAP_TXN_BEGIN, &si->si_refreshTxn );
+               }
+               si->si_refreshCount++;
+       }
 
        slap_op_time( &op->o_time, &op->o_tincr );
        switch ( syncstate ) {