From 21bf33b0e8e044d6722a9c3ef4a3961d71465d77 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Thu, 27 Aug 2015 14:47:04 +0100 Subject: [PATCH] ITS#8226 limit size of read txns in searches --- doc/man/man5/slapd-mdb.5 | 10 ++++++ servers/slapd/back-mdb/back-mdb.h | 4 +++ servers/slapd/back-mdb/config.c | 29 +++++++--------- servers/slapd/back-mdb/init.c | 1 + servers/slapd/back-mdb/search.c | 58 ++++++++++++++++++------------- 5 files changed, 61 insertions(+), 41 deletions(-) diff --git a/doc/man/man5/slapd-mdb.5 b/doc/man/man5/slapd-mdb.5 index 789b0914f3..d742aa1fc5 100644 --- a/doc/man/man5/slapd-mdb.5 +++ b/doc/man/man5/slapd-mdb.5 @@ -167,6 +167,16 @@ Specify the file protection mode that newly created database files should have. The default is 0600. .TP +.BI rtxnsize \ +Specify the maximum number of entries to process in a single read +transaction when executing a large search. Long-lived read transactions +prevent old database pages from being reused in write transactions, and +so can cause significant growth of the database file when there is +heavy write traffic. This setting causes the read transaction in +large searches to be released and reacquired after the given number +of entries has been read, to give writers the opportunity to +reclaim old database pages. The default is 10000. +.TP .BI searchstack \ Specify the depth of the stack used for search filter evaluation. Search filters are evaluated on a stack to accommodate nested AND / OR diff --git a/servers/slapd/back-mdb/back-mdb.h b/servers/slapd/back-mdb/back-mdb.h index 8fe2561b66..9e8025bf1d 100644 --- a/servers/slapd/back-mdb/back-mdb.h +++ b/servers/slapd/back-mdb/back-mdb.h @@ -47,6 +47,9 @@ LDAP_BEGIN_DECL /* Default to 10MB max */ #define DEFAULT_MAPSIZE (10*1048576) +/* Most users will never see this */ +#define DEFAULT_RTXN_SIZE 10000 + #define MDB_MONITOR_IDX typedef struct mdb_monitor_t { @@ -76,6 +79,7 @@ struct mdb_info { int mi_search_stack_depth; int mi_readers; + uint32_t mi_rtxn_size; int mi_txn_cp; uint32_t mi_txn_cp_min; uint32_t mi_txn_cp_kbyte; diff --git a/servers/slapd/back-mdb/config.c b/servers/slapd/back-mdb/config.c index c1995176e8..077281131e 100644 --- a/servers/slapd/back-mdb/config.c +++ b/servers/slapd/back-mdb/config.c @@ -40,7 +40,6 @@ enum { MDB_MAXSIZE, MDB_MODE, MDB_SSTACK, - MDB_MAXENTSZ }; static ConfigTable mdbcfg[] = { @@ -67,8 +66,9 @@ static ConfigTable mdbcfg[] = { "DESC 'Attribute index parameters' " "EQUALITY caseIgnoreMatch " "SYNTAX OMsDirectoryString )", NULL, NULL }, - { "maxentrysize", "size", 2, 2, 0, ARG_ULONG|ARG_MAGIC|MDB_MAXENTSZ, - mdb_cf_gen, "( OLcfgDbAt:12.4 NAME 'olcDbMaxEntrySize' " + { "maxentrysize", "size", 2, 2, 0, ARG_ULONG|ARG_OFFSET, + (void *)offsetof(struct mdb_info, mi_maxentrysize), + "( OLcfgDbAt:12.4 NAME 'olcDbMaxEntrySize' " "DESC 'Maximum size of an entry in bytes' " "SYNTAX OMsInteger SINGLE-VALUE )", NULL, NULL }, { "maxreaders", "num", 2, 2, 0, ARG_UINT|ARG_MAGIC|MDB_MAXREADERS, @@ -83,6 +83,11 @@ static ConfigTable mdbcfg[] = { mdb_cf_gen, "( OLcfgDbAt:0.3 NAME 'olcDbMode' " "DESC 'Unix permissions of database files' " "SYNTAX OMsDirectoryString SINGLE-VALUE )", NULL, NULL }, + { "rtxnsize", "entries", 2, 2, 0, ARG_UINT|ARG_OFFSET, + (void *)offsetof(struct mdb_info, mi_rtxn_size), + "( OLcfgDbAt:12.5 NAME 'olcDbRtxnSize' " + "DESC 'Number of entries to process in one read transaction' " + "SYNTAX OMsInteger SINGLE-VALUE )", NULL, NULL }, { "searchstack", "depth", 2, 2, 0, ARG_INT|ARG_MAGIC|MDB_SSTACK, mdb_cf_gen, "( OLcfgDbAt:1.9 NAME 'olcDbSearchStack' " "DESC 'Depth of search stack in IDLs' " @@ -100,7 +105,7 @@ static ConfigOCs mdbocs[] = { "MUST olcDbDirectory " "MAY ( olcDbCheckpoint $ olcDbEnvFlags $ " "olcDbNoSync $ olcDbIndex $ olcDbMaxReaders $ olcDbMaxSize $ " - "olcDbMode $ olcDbSearchStack $ olcDbMaxEntrySize ) )", + "olcDbMode $ olcDbSearchStack $ olcDbMaxEntrySize $ olcDbRtxnSize ) )", Cft_Database, mdbcfg }, { NULL, 0, NULL } }; @@ -334,10 +339,6 @@ mdb_cf_gen( ConfigArgs *c ) c->value_int = mdb->mi_search_stack_depth; break; - case MDB_MAXENTSZ: - c->value_ulong = mdb->mi_maxentrysize; - break; - case MDB_MAXREADERS: c->value_int = mdb->mi_readers; break; @@ -345,6 +346,10 @@ mdb_cf_gen( ConfigArgs *c ) case MDB_MAXSIZE: c->value_ulong = mdb->mi_mapsize; break; + + case MDB_RTXNSIZE: + c->value_int = mdb->mi_rtxn_size; + break; } return rc; } else if ( c->op == LDAP_MOD_DELETE ) { @@ -364,10 +369,6 @@ mdb_cf_gen( ConfigArgs *c ) case MDB_MAXSIZE: break; - case MDB_MAXENTSZ: - mdb->mi_maxentrysize = 0; - break; - case MDB_CHKPT: if ( mdb->mi_txn_cp_task ) { struct re_s *re = mdb->mi_txn_cp_task; @@ -670,10 +671,6 @@ mdb_cf_gen( ConfigArgs *c ) mdb->mi_search_stack_depth = c->value_int; break; - case MDB_MAXENTSZ: - mdb->mi_maxentrysize = c->value_ulong; - break; - case MDB_MAXREADERS: mdb->mi_readers = c->value_int; if ( mdb->mi_flags & MDB_IS_OPEN ) { diff --git a/servers/slapd/back-mdb/init.c b/servers/slapd/back-mdb/init.c index e2f26eb295..1089e0f01c 100644 --- a/servers/slapd/back-mdb/init.c +++ b/servers/slapd/back-mdb/init.c @@ -62,6 +62,7 @@ mdb_db_init( BackendDB *be, ConfigReply *cr ) mdb->mi_search_stack = NULL; mdb->mi_mapsize = DEFAULT_MAPSIZE; + mdb->mi_rtxn_size = DEFAULT_RTXN_SIZE; be->be_private = mdb; be->be_cf_ocs = be->bd_info->bi_cf_ocs; diff --git a/servers/slapd/back-mdb/search.c b/servers/slapd/back-mdb/search.c index 14a2c6bc4f..502312add2 100644 --- a/servers/slapd/back-mdb/search.c +++ b/servers/slapd/back-mdb/search.c @@ -327,6 +327,7 @@ typedef struct ww_ctx { ID key; MDB_val data; int flag; + int nentries; } ww_ctx; /* ITS#7904 if we get blocked while writing results to client, @@ -339,21 +340,28 @@ typedef struct ww_ctx { * case return an LDAP_BUSY error - let the client know this search * couldn't succeed, but might succeed on a retry. */ +static void +mdb_rtxn_snap( Operation *op, ww_ctx *ww ) +{ + /* save cursor position and release read txn */ + if ( ww->mcd ) { + MDB_val key, data; + mdb_cursor_get( ww->mcd, &key, &data, MDB_GET_CURRENT ); + memcpy( &ww->key, key.mv_data, sizeof(ID) ); + ww->data.mv_size = data.mv_size; + ww->data.mv_data = op->o_tmpalloc( data.mv_size, op->o_tmpmemctx ); + memcpy(ww->data.mv_data, data.mv_data, data.mv_size); + } + mdb_txn_reset( ww->txn ); + ww->flag = 1; +} + static void mdb_writewait( Operation *op, slap_callback *sc ) { ww_ctx *ww = sc->sc_private; if ( !ww->flag ) { - if ( ww->mcd ) { - MDB_val key, data; - mdb_cursor_get( ww->mcd, &key, &data, MDB_GET_CURRENT ); - memcpy( &ww->key, key.mv_data, sizeof(ID) ); - ww->data.mv_size = data.mv_size; - ww->data.mv_data = op->o_tmpalloc( data.mv_size, op->o_tmpmemctx ); - memcpy(ww->data.mv_data, data.mv_data, data.mv_size); - } - mdb_txn_reset( ww->txn ); - ww->flag = 1; + mdb_rtxn_snap( op, ww ); } } @@ -1048,14 +1056,6 @@ notfound: ber_bvarray_free( erefs ); rs->sr_ref = NULL; - if ( wwctx.flag ) { - rs->sr_err = mdb_waitfixup( op, &wwctx, mci, mcd, &isc ); - if ( rs->sr_err ) { - send_ldap_result( op, rs ); - goto done; - } - } - goto loop_continue; } @@ -1110,13 +1110,6 @@ notfound: } goto done; } - if ( wwctx.flag ) { - rs->sr_err = mdb_waitfixup( op, &wwctx, mci, mcd, &isc ); - if ( rs->sr_err ) { - send_ldap_result( op, rs ); - goto done; - } - } } } else { @@ -1127,6 +1120,21 @@ notfound: } loop_continue: + if ( !wwctx.flag && mdb->mi_rtxn_size ) { + wwctx.nentries++; + if ( wwctx.nentries >= mdb->mi_rtxn_size ) { + wwctx.nentries = 0; + mdb_rtxn_snap( op, &wwctx ); + } + } + if ( wwctx.flag ) { + rs->sr_err = mdb_waitfixup( op, &wwctx, mci, mcd, &isc ); + if ( rs->sr_err ) { + send_ldap_result( op, rs ); + goto done; + } + } + if( e != NULL ) { if ( e != base ) mdb_entry_return( op, e ); -- 2.39.5