From c647a06441523eaed981dcf8cd5e40325399d903 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Thu, 6 Oct 2011 19:15:10 -0700 Subject: [PATCH] More for cursor reuse --- servers/slapd/back-mdb/add.c | 51 ++++-- servers/slapd/back-mdb/attr.c | 4 + servers/slapd/back-mdb/back-mdb.h | 26 +++- servers/slapd/back-mdb/bind.c | 2 +- servers/slapd/back-mdb/compare.c | 2 +- servers/slapd/back-mdb/delete.c | 16 +- servers/slapd/back-mdb/dn2entry.c | 3 +- servers/slapd/back-mdb/filterindex.c | 4 +- servers/slapd/back-mdb/index.c | 55 ++++--- servers/slapd/back-mdb/modify.c | 4 +- servers/slapd/back-mdb/modrdn.c | 27 +++- servers/slapd/back-mdb/proto-mdb.h | 16 +- servers/slapd/back-mdb/search.c | 4 +- servers/slapd/back-mdb/tools.c | 224 ++++++++++++++++----------- 14 files changed, 281 insertions(+), 157 deletions(-) diff --git a/servers/slapd/back-mdb/add.c b/servers/slapd/back-mdb/add.c index 652bb1009b..3bfe12eb76 100644 --- a/servers/slapd/back-mdb/add.c +++ b/servers/slapd/back-mdb/add.c @@ -32,7 +32,9 @@ mdb_add(Operation *op, SlapReply *rs ) AttributeDescription *children = slap_schema.si_ad_children; AttributeDescription *entry = slap_schema.si_ad_entry; MDB_txn *txn = NULL; - ID eid = NOID, pid = 0; + MDB_cursor *mc = NULL; + MDB_cursor *mcd; + ID eid, pid = 0; mdb_op_info opinfo = {{{ 0 }}}, *moi = &opinfo; int subentry; @@ -141,8 +143,18 @@ txnReturn: dnParent( &op->ora_e->e_nname, &pdn ); } + rs->sr_err = mdb_cursor_open( txn, mdb->mi_dn2id, &mcd ); + if( rs->sr_err != 0 ) { + Debug( LDAP_DEBUG_TRACE, + LDAP_XSTRING(mdb_add) ": mdb_cursor_open failed (%d)\n", + rs->sr_err, 0, 0 ); + rs->sr_err = LDAP_OTHER; + rs->sr_text = "internal error"; + goto return_results; + } + /* get entry or parent */ - rs->sr_err = mdb_dn2entry( op, txn, &op->ora_e->e_nname, &p, 1 ); + rs->sr_err = mdb_dn2entry( op, txn, mcd, &op->ora_e->e_nname, &p, 1 ); switch( rs->sr_err ) { case 0: rs->sr_err = LDAP_ALREADY_EXISTS; @@ -305,21 +317,30 @@ txnReturn: goto return_results;; } - if ( eid == NOID ) { - rs->sr_err = mdb_next_id( op->o_bd, txn, &eid ); - if( rs->sr_err != 0 ) { - Debug( LDAP_DEBUG_TRACE, - LDAP_XSTRING(mdb_add) ": next_id failed (%d)\n", - rs->sr_err, 0, 0 ); - rs->sr_err = LDAP_OTHER; - rs->sr_text = "internal error"; - goto return_results; - } - op->ora_e->e_id = eid; + rs->sr_err = mdb_cursor_open( txn, mdb->mi_id2entry, &mc ); + if( rs->sr_err != 0 ) { + Debug( LDAP_DEBUG_TRACE, + LDAP_XSTRING(mdb_add) ": mdb_cursor_open failed (%d)\n", + rs->sr_err, 0, 0 ); + rs->sr_err = LDAP_OTHER; + rs->sr_text = "internal error"; + goto return_results; + } + + rs->sr_err = mdb_next_id( op->o_bd, mc, &eid ); + if( rs->sr_err != 0 ) { + Debug( LDAP_DEBUG_TRACE, + LDAP_XSTRING(mdb_add) ": next_id failed (%d)\n", + rs->sr_err, 0, 0 ); + rs->sr_err = LDAP_OTHER; + rs->sr_text = "internal error"; + goto return_results; } + op->ora_e->e_id = eid; /* dn2id index */ - rs->sr_err = mdb_dn2id_add( op, txn, pid, op->ora_e ); + rs->sr_err = mdb_dn2id_add( op, mcd, mcd, pid, op->ora_e ); + mdb_cursor_close( mcd ); if ( rs->sr_err != 0 ) { Debug( LDAP_DEBUG_TRACE, LDAP_XSTRING(mdb_add) ": dn2id_add failed: %s (%d)\n", @@ -347,7 +368,7 @@ txnReturn: } /* id2entry index */ - rs->sr_err = mdb_id2entry_add( op, txn, op->ora_e ); + rs->sr_err = mdb_id2entry_add( op, txn, mc, op->ora_e ); if ( rs->sr_err != 0 ) { Debug( LDAP_DEBUG_TRACE, LDAP_XSTRING(mdb_add) ": id2entry_add failed\n", diff --git a/servers/slapd/back-mdb/attr.c b/servers/slapd/back-mdb/attr.c index f4f8c56b61..3cb2138161 100644 --- a/servers/slapd/back-mdb/attr.c +++ b/servers/slapd/back-mdb/attr.c @@ -358,6 +358,10 @@ mdb_attr_index_config( #ifdef LDAP_COMP_MATCH a->ai_cr = NULL; #endif + a->ai_cursor = NULL; + a->ai_flist = NULL; + a->ai_clist = NULL; + a->ai_root = NULL; a->ai_desc = ad; a->ai_dbi = 0; diff --git a/servers/slapd/back-mdb/back-mdb.h b/servers/slapd/back-mdb/back-mdb.h index 8b2e9f65ce..a67d939813 100644 --- a/servers/slapd/back-mdb/back-mdb.h +++ b/servers/slapd/back-mdb/back-mdb.h @@ -23,6 +23,8 @@ LDAP_BEGIN_DECL +#define MDB_TOOL_IDL_CACHING 1 + #define DN_BASE_PREFIX SLAP_INDEX_EQUALITY_PREFIX #define DN_ONE_PREFIX '%' #define DN_SUBTREE_PREFIX '@' @@ -142,6 +144,10 @@ typedef struct mdb_attrinfo { #ifdef LDAP_COMP_MATCH ComponentReference* ai_cr; /*component indexing*/ #endif + Avlnode *ai_root; /* for tools */ + void *ai_flist; /* for tools */ + void *ai_clist; /* for tools */ + MDB_cursor *ai_cursor; /* for tools */ int ai_idx; /* position in AI array */ MDB_dbi ai_dbi; } AttrInfo; @@ -158,9 +164,27 @@ typedef struct AttrList { Attribute *attr; } AttrList; -typedef struct IndexRec { +#ifndef CACHELINE +#define CACHELINE 64 +#endif + +typedef struct IndexRbody { AttrInfo *ai; AttrList *attrs; + void *tptr; + int i; +} IndexRbody; + +typedef struct IndexRec { + union { + IndexRbody irb; +#define ir_ai iru.irb.ai +#define ir_attrs iru.irb.attrs +#define ir_tptr iru.irb.tptr +#define ir_i iru.irb.i + /* cache line alignment */ + char pad[(sizeof(IndexRbody)+CACHELINE-1) & (!CACHELINE-1)]; + } iru; } IndexRec; #define MAXRDNS SLAP_LDAPDN_MAXLEN/4 diff --git a/servers/slapd/back-mdb/bind.c b/servers/slapd/back-mdb/bind.c index 9a00b99848..24ddf86fc0 100644 --- a/servers/slapd/back-mdb/bind.c +++ b/servers/slapd/back-mdb/bind.c @@ -67,7 +67,7 @@ mdb_bind( Operation *op, SlapReply *rs ) rtxn = moi->moi_txn; /* get entry with reader lock */ - rs->sr_err = mdb_dn2entry( op, rtxn, &op->o_req_ndn, &e, 0 ); + rs->sr_err = mdb_dn2entry( op, rtxn, NULL, &op->o_req_ndn, &e, 0 ); switch(rs->sr_err) { case MDB_NOTFOUND: diff --git a/servers/slapd/back-mdb/compare.c b/servers/slapd/back-mdb/compare.c index 9ed7f9bde5..26eafbe00d 100644 --- a/servers/slapd/back-mdb/compare.c +++ b/servers/slapd/back-mdb/compare.c @@ -43,7 +43,7 @@ mdb_compare( Operation *op, SlapReply *rs ) rtxn = moi->moi_txn; /* get entry */ - rs->sr_err = mdb_dn2entry( op, rtxn, &op->o_req_ndn, &e, 1 ); + rs->sr_err = mdb_dn2entry( op, rtxn, NULL, &op->o_req_ndn, &e, 1 ); switch( rs->sr_err ) { case MDB_NOTFOUND: case 0: diff --git a/servers/slapd/back-mdb/delete.c b/servers/slapd/back-mdb/delete.c index a557fc05bc..116bcdc411 100644 --- a/servers/slapd/back-mdb/delete.c +++ b/servers/slapd/back-mdb/delete.c @@ -33,6 +33,7 @@ mdb_delete( Operation *op, SlapReply *rs ) AttributeDescription *children = slap_schema.si_ad_children; AttributeDescription *entry = slap_schema.si_ad_entry; MDB_txn *txn = NULL; + MDB_cursor *mc; mdb_op_info opinfo = {{{ 0 }}}, *moi = &opinfo; LDAPControl **preread_ctrl = NULL; @@ -117,8 +118,14 @@ txnReturn: dnParent( &op->o_req_ndn, &pdn ); } + rs->sr_err = mdb_cursor_open( txn, mdb->mi_dn2id, &mc ); + if ( rs->sr_err ) { + rs->sr_err = LDAP_OTHER; + rs->sr_text = "internal error"; + goto return_results; + } /* get parent */ - rs->sr_err = mdb_dn2entry( op, txn, &pdn, &p, 1 ); + rs->sr_err = mdb_dn2entry( op, txn, mc, &pdn, &p, 1 ); switch( rs->sr_err ) { case 0: case MDB_NOTFOUND: @@ -161,7 +168,7 @@ txnReturn: } /* get entry */ - rs->sr_err = mdb_dn2entry( op, txn, &op->o_req_ndn, &e, 0 ); + rs->sr_err = mdb_dn2entry( op, txn, mc, &op->o_req_ndn, &e, 0 ); switch( rs->sr_err ) { case MDB_NOTFOUND: e = p; @@ -326,7 +333,8 @@ txnReturn: } /* delete from dn2id */ - rs->sr_err = mdb_dn2id_delete( op, txn, p->e_id, e ); + rs->sr_err = mdb_dn2id_delete( op, mc, e->e_id ); + mdb_cursor_close( mc ); if ( rs->sr_err != 0 ) { Debug(LDAP_DEBUG_TRACE, "<=- " LDAP_XSTRING(mdb_delete) ": dn2id failed: " @@ -356,7 +364,7 @@ txnReturn: BER_BVZERO( &vals[1] ); rs->sr_err = mdb_index_values( op, txn, slap_schema.si_ad_entryCSN, vals, 0, SLAP_INDEX_ADD_OP ); - if ( rs->sr_err != LDAP_SUCCESS ) { + if ( rs->sr_err != LDAP_SUCCESS ) { rs->sr_text = "entryCSN index update failed"; rs->sr_err = LDAP_OTHER; goto return_results; diff --git a/servers/slapd/back-mdb/dn2entry.c b/servers/slapd/back-mdb/dn2entry.c index e12a9da2a6..26bf846437 100644 --- a/servers/slapd/back-mdb/dn2entry.c +++ b/servers/slapd/back-mdb/dn2entry.c @@ -31,6 +31,7 @@ int mdb_dn2entry( Operation *op, MDB_txn *tid, + MDB_cursor *m2, struct berval *dn, Entry **e, int matched ) @@ -46,7 +47,7 @@ mdb_dn2entry( *e = NULL; - rc = mdb_dn2id( op, tid, dn, &id, &mbv, &nmbv ); + rc = mdb_dn2id( op, tid, m2, dn, &id, &mbv, &nmbv ); if ( rc ) { if ( matched ) { rc2 = mdb_cursor_open( tid, mdb->mi_id2entry, &mc ); diff --git a/servers/slapd/back-mdb/filterindex.c b/servers/slapd/back-mdb/filterindex.c index 3e3a87b7ef..da025c0508 100644 --- a/servers/slapd/back-mdb/filterindex.c +++ b/servers/slapd/back-mdb/filterindex.c @@ -489,7 +489,7 @@ ext_candidates( MDB_IDL_ZERO( ids ); if ( mra->ma_rule == slap_schema.si_mr_distinguishedNameMatch ) { base: - rc = mdb_dn2id( op, rtxn, &mra->ma_value, &id, NULL, NULL ); + rc = mdb_dn2id( op, rtxn, NULL, &mra->ma_value, &id, NULL, NULL ); if ( rc == MDB_SUCCESS ) { mdb_idl_insert( ids, id ); } @@ -690,7 +690,7 @@ equality_candidates( if ( ava->aa_desc == slap_schema.si_ad_entryDN ) { ID id; - rc = mdb_dn2id( op, rtxn, &ava->aa_value, &id, NULL, NULL ); + rc = mdb_dn2id( op, rtxn, NULL, &ava->aa_value, &id, NULL, NULL ); if ( rc == LDAP_SUCCESS ) { /* exactly one ID can match */ ids[0] = 1; diff --git a/servers/slapd/back-mdb/index.c b/servers/slapd/back-mdb/index.c index 1c6a23871f..58faf40444 100644 --- a/servers/slapd/back-mdb/index.c +++ b/servers/slapd/back-mdb/index.c @@ -164,7 +164,7 @@ done: static int indexer( Operation *op, MDB_txn *txn, - MDB_dbi dbi, + struct mdb_attrinfo *ai, AttributeDescription *ad, struct berval *atname, BerVarray vals, @@ -174,21 +174,26 @@ static int indexer( { int rc, i; struct berval *keys; - MDB_cursor *mc; + MDB_cursor *mc = ai->ai_cursor; mdb_idl_keyfunc *keyfunc; char *err; assert( mask != 0 ); - err = "c_open"; - rc = mdb_cursor_open( txn, dbi, &mc ); - if ( rc ) goto done; + if ( !mc ) { + err = "c_open"; + rc = mdb_cursor_open( txn, ai->ai_dbi, &mc ); + if ( rc ) goto done; + if ( slapMode & SLAP_TOOL_QUICK ) + ai->ai_cursor = mc; + } if ( opid == SLAP_INDEX_ADD_OP ) { #ifdef MDB_TOOL_IDL_CACHING - if ( slapMode & SLAP_TOOL_QUICK ) + if ( slapMode & SLAP_TOOL_QUICK ) { keyfunc = mdb_tool_idl_add; - else + mc = (MDB_cursor *)ai; + } else #endif keyfunc = mdb_idl_insert_keys; } else @@ -262,7 +267,8 @@ static int indexer( } done: - mdb_cursor_close( mc ); + if ( !(slapMode & SLAP_TOOL_QUICK)) + mdb_cursor_close( mc ); switch( rc ) { /* The callers all know how to deal with these results */ case 0: @@ -310,7 +316,7 @@ static int index_at_values( if ( ai->ai_cr ) { ComponentReference *cr; for( cr = ai->ai_cr ; cr ; cr = cr->cr_next ) { - rc = indexer( op, txn, ai->ai_dbi, cr->cr_ad, &type->sat_cname, + rc = indexer( op, txn, ai, cr->cr_ad, &type->sat_cname, cr->cr_nvals, id, ixop, cr->cr_indexmask ); } @@ -328,7 +334,7 @@ static int index_at_values( */ mask = ai->ai_newmask ? ai->ai_newmask : ai->ai_indexmask; if( mask ) { - rc = indexer( op, txn, ai->ai_dbi, ad, &type->sat_cname, + rc = indexer( op, txn, ai, ad, &type->sat_cname, vals, id, ixop, mask ); if( rc ) return rc; @@ -349,7 +355,7 @@ static int index_at_values( else mask = ai->ai_newmask ? ai->ai_newmask : ai->ai_indexmask; if ( mask ) { - rc = indexer( op, txn, ai->ai_dbi, desc, &desc->ad_cname, + rc = indexer( op, txn, ai, desc, &desc->ad_cname, vals, id, ixop, mask ); if( rc ) { @@ -405,11 +411,11 @@ mdb_index_recset( if( type->sat_ad ) { slot = mdb_attr_slot( mdb, type->sat_ad, NULL ); if ( slot >= 0 ) { - ir[slot].ai = mdb->mi_attrs[slot]; + ir[slot].ir_ai = mdb->mi_attrs[slot]; al = ch_malloc( sizeof( AttrList )); al->attr = a; - al->next = ir[slot].attrs; - ir[slot].attrs = al; + al->next = ir[slot].ir_attrs; + ir[slot].ir_attrs = al; } } if( tags->bv_len ) { @@ -419,11 +425,11 @@ mdb_index_recset( if( desc ) { slot = mdb_attr_slot( mdb, desc, NULL ); if ( slot >= 0 ) { - ir[slot].ai = mdb->mi_attrs[slot]; + ir[slot].ir_ai = mdb->mi_attrs[slot]; al = ch_malloc( sizeof( AttrList )); al->attr = a; - al->next = ir[slot].attrs; - ir[slot].attrs = al; + al->next = ir[slot].ir_attrs; + ir[slot].ir_attrs = al; } } } @@ -433,6 +439,7 @@ mdb_index_recset( /* Apply the indices for the recset */ int mdb_index_recrun( Operation *op, + MDB_txn *txn, struct mdb_info *mdb, IndexRec *ir0, ID id, @@ -446,15 +453,15 @@ int mdb_index_recrun( if ( id == 0 ) return 0; - for (i=base; imi_nattrs; i+=slap_tool_thread_max) { + for (i=base; imi_nattrs; i+=slap_tool_thread_max-1) { ir = ir0 + i; - if ( !ir->ai ) continue; - while (( al = ir->attrs )) { - ir->attrs = al->next; - rc = indexer( op, NULL, ir->ai->ai_dbi, ir->ai->ai_desc, - &ir->ai->ai_desc->ad_type->sat_cname, + if ( !ir->ir_ai ) continue; + while (( al = ir->ir_attrs )) { + ir->ir_attrs = al->next; + rc = indexer( op, txn, ir->ir_ai, ir->ir_ai->ai_desc, + &ir->ir_ai->ai_desc->ad_type->sat_cname, al->attr->a_nvals, id, SLAP_INDEX_ADD_OP, - ir->ai->ai_indexmask ); + ir->ir_ai->ai_indexmask ); free( al ); if ( rc ) break; } diff --git a/servers/slapd/back-mdb/modify.c b/servers/slapd/back-mdb/modify.c index 91ece764b8..ced75aa4b0 100644 --- a/servers/slapd/back-mdb/modify.c +++ b/servers/slapd/back-mdb/modify.c @@ -480,7 +480,7 @@ txnReturn: txn = moi->moi_txn; /* get entry or ancestor */ - rs->sr_err = mdb_dn2entry( op, txn, &op->o_req_ndn, &e, 1 ); + rs->sr_err = mdb_dn2entry( op, txn, NULL, &op->o_req_ndn, &e, 1 ); if ( rs->sr_err != 0 ) { Debug( LDAP_DEBUG_TRACE, @@ -585,7 +585,7 @@ txnReturn: } /* change the entry itself */ - rs->sr_err = mdb_id2entry_update( op, txn, &dummy ); + rs->sr_err = mdb_id2entry_update( op, txn, NULL, &dummy ); if ( rs->sr_err != 0 ) { Debug( LDAP_DEBUG_TRACE, LDAP_XSTRING(mdb_modify) ": id2entry update failed " "(%d)\n", diff --git a/servers/slapd/back-mdb/modrdn.c b/servers/slapd/back-mdb/modrdn.c index 08d615e261..9351e3213c 100644 --- a/servers/slapd/back-mdb/modrdn.c +++ b/servers/slapd/back-mdb/modrdn.c @@ -35,6 +35,7 @@ mdb_modrdn( Operation *op, SlapReply *rs ) char textbuf[SLAP_TEXT_BUFLEN]; size_t textlen = sizeof textbuf; MDB_txn *txn = NULL; + MDB_cursor *mc; struct mdb_op_info opinfo = {{{ 0 }}}, *moi = &opinfo; Entry dummy = {0}; @@ -135,7 +136,17 @@ txnReturn: /* Make sure parent entry exist and we can write its * children. */ - rs->sr_err = mdb_dn2entry( op, txn, &p_ndn, &p, 0 ); + rs->sr_err = mdb_cursor_open( txn, mdb->mi_dn2id, &mc ); + if ( rs->sr_err != 0 ) { + Debug(LDAP_DEBUG_TRACE, + "<=- " LDAP_XSTRING(mdb_modrdn) + ": cursor_open failed: %s (%d)\n", + mdb_strerror(rs->sr_err), rs->sr_err, 0 ); + rs->sr_err = LDAP_OTHER; + rs->sr_text = "DN cursor_open failed"; + goto return_results; + } + rs->sr_err = mdb_dn2entry( op, txn, mc, &p_ndn, &p, 0 ); switch( rs->sr_err ) { case MDB_NOTFOUND: Debug( LDAP_DEBUG_TRACE, LDAP_XSTRING(mdb_modrdn) @@ -189,7 +200,7 @@ txnReturn: p_dn.bv_val, 0, 0 ); /* get entry */ - rs->sr_err = mdb_dn2entry( op, txn, &op->o_req_ndn, &e, 0 ); + rs->sr_err = mdb_dn2entry( op, txn, mc, &op->o_req_ndn, &e, 0 ); switch( rs->sr_err ) { case MDB_NOTFOUND: e = p; @@ -311,8 +322,7 @@ txnReturn: goto return_results; } /* Get Entry with dn=newSuperior. Does newSuperior exist? */ - - rs->sr_err = mdb_dn2entry( op, txn, np_ndn, &np, 0 ); + rs->sr_err = mdb_dn2entry( op, txn, NULL, np_ndn, &np, 0 ); switch( rs->sr_err ) { case 0: @@ -423,7 +433,7 @@ txnReturn: new_ndn.bv_val, 0, 0 ); /* Shortcut the search */ - rs->sr_err = mdb_dn2id ( op, txn, &new_ndn, &nid, NULL, NULL ); + rs->sr_err = mdb_dn2id ( op, txn, NULL, &new_ndn, &nid, NULL, NULL ); switch( rs->sr_err ) { case MDB_NOTFOUND: break; @@ -461,7 +471,7 @@ txnReturn: } /* delete old DN */ - rs->sr_err = mdb_dn2id_delete( op, txn, p->e_id, e ); + rs->sr_err = mdb_dn2id_delete( op, mc, e->e_id ); if ( rs->sr_err != 0 ) { Debug(LDAP_DEBUG_TRACE, "<=- " LDAP_XSTRING(mdb_modrdn) @@ -479,7 +489,7 @@ txnReturn: dummy.e_attrs = NULL; /* add new DN */ - rs->sr_err = mdb_dn2id_add( op, txn, np ? np->e_id : p->e_id, &dummy ); + rs->sr_err = mdb_dn2id_add( op, mc, mc, np ? np->e_id : p->e_id, &dummy ); if ( rs->sr_err != 0 ) { Debug(LDAP_DEBUG_TRACE, "<=- " LDAP_XSTRING(mdb_modrdn) @@ -505,7 +515,7 @@ txnReturn: } /* id2entry index */ - rs->sr_err = mdb_id2entry_update( op, txn, &dummy ); + rs->sr_err = mdb_id2entry_update( op, txn, NULL, &dummy ); if ( rs->sr_err != 0 ) { Debug(LDAP_DEBUG_TRACE, "<=- " LDAP_XSTRING(mdb_modrdn) @@ -532,6 +542,7 @@ txnReturn: rs->sr_text = "internal error"; goto return_results; } + } else { parent_is_leaf = 1; } mdb_entry_return( op, p ); diff --git a/servers/slapd/back-mdb/proto-mdb.h b/servers/slapd/back-mdb/proto-mdb.h index 3c2e255b25..aef1405dde 100644 --- a/servers/slapd/back-mdb/proto-mdb.h +++ b/servers/slapd/back-mdb/proto-mdb.h @@ -59,7 +59,7 @@ int mdb_back_init_cf( BackendInfo *bi ); * dn2entry.c */ -int mdb_dn2entry LDAP_P(( Operation *op, MDB_txn *tid, +int mdb_dn2entry LDAP_P(( Operation *op, MDB_txn *tid, MDB_cursor *mc, struct berval *dn, Entry **e, int matched )); /* @@ -69,6 +69,7 @@ int mdb_dn2entry LDAP_P(( Operation *op, MDB_txn *tid, int mdb_dn2id( Operation *op, MDB_txn *txn, + MDB_cursor *mc, struct berval *ndn, ID *id, struct berval *matched, @@ -76,15 +77,15 @@ int mdb_dn2id( int mdb_dn2id_add( Operation *op, - MDB_txn *tid, + MDB_cursor *mcp, + MDB_cursor *mcd, ID pid, Entry *e ); int mdb_dn2id_delete( Operation *op, - MDB_txn *tid, - ID pid, - Entry *e ); + MDB_cursor *mc, + ID id ); int mdb_dn2id_children( Operation *op, @@ -154,11 +155,13 @@ int mdb_filter_candidates( int mdb_id2entry_add( Operation *op, MDB_txn *tid, + MDB_cursor *mc, Entry *e ); int mdb_id2entry_update( Operation *op, MDB_txn *tid, + MDB_cursor *mc, Entry *e ); int mdb_id2entry_delete( @@ -263,6 +266,7 @@ mdb_index_recset LDAP_P(( extern int mdb_index_recrun LDAP_P(( Operation *op, + MDB_txn *txn, struct mdb_info *mdb, IndexRec *ir, ID id, @@ -293,7 +297,7 @@ mdb_key_read( * nextid.c */ -int mdb_next_id( BackendDB *be, MDB_txn *tid, ID *id ); +int mdb_next_id( BackendDB *be, MDB_cursor *mc, ID *id ); /* * modify.c diff --git a/servers/slapd/back-mdb/search.c b/servers/slapd/back-mdb/search.c index 5bac5501c9..fd317fa8bb 100644 --- a/servers/slapd/back-mdb/search.c +++ b/servers/slapd/back-mdb/search.c @@ -98,7 +98,7 @@ static Entry * deref_base ( break; } - rs->sr_err = mdb_dn2entry( op, txn, &ndn, &e, 0 ); + rs->sr_err = mdb_dn2entry( op, txn, NULL, &ndn, &e, 0 ); if (rs->sr_err) { rs->sr_err = LDAP_ALIAS_PROBLEM; rs->sr_text = "aliasedObject not found"; @@ -327,7 +327,7 @@ mdb_search( Operation *op, SlapReply *rs ) } dn2entry_retry: /* get entry with reader lock */ - rs->sr_err = mdb_dn2entry( op, ltid, &op->o_req_ndn, &e, 1 ); + rs->sr_err = mdb_dn2entry( op, ltid, NULL, &op->o_req_ndn, &e, 1 ); switch(rs->sr_err) { case MDB_NOTFOUND: diff --git a/servers/slapd/back-mdb/tools.c b/servers/slapd/back-mdb/tools.c index e4da95ec94..2d733849a5 100644 --- a/servers/slapd/back-mdb/tools.c +++ b/servers/slapd/back-mdb/tools.c @@ -45,9 +45,6 @@ typedef struct mdb_tool_idl_cache { #define WAS_FOUND 0x01 #define WAS_RANGE 0x02 -static mdb_tool_idl_cache_entry *mdb_tool_idl_free_list; -static Avlnode *mdb_tool_roots[MDB_INDICES]; - #define MDB_TOOL_IDL_FLUSH(be, txn) mdb_tool_idl_flush(be, txn) #else #define MDB_TOOL_IDL_FLUSH(be, txn) @@ -55,6 +52,7 @@ static Avlnode *mdb_tool_roots[MDB_INDICES]; static MDB_txn *txn = NULL, *txi = NULL; static MDB_cursor *cursor = NULL, *idcursor = NULL; +static MDB_cursor *mcp = NULL, *mcd = NULL; static MDB_val key, data; static ID previd = NOID; @@ -73,17 +71,16 @@ static int tool_scope; static Filter *tool_filter; static Entry *tool_next_entry; -#if 0 static ID mdb_tool_ix_id; static Operation *mdb_tool_ix_op; -static int *mdb_tool_index_threads, mdb_tool_index_tcount; -static void *mdb_tool_index_rec; +static MDB_txn *mdb_tool_ix_txn; +static int mdb_tool_index_tcount, mdb_tool_threads; +static IndexRec *mdb_tool_index_rec; static struct mdb_info *mdb_tool_info; static ldap_pvt_thread_mutex_t mdb_tool_index_mutex; static ldap_pvt_thread_cond_t mdb_tool_index_cond_main; static ldap_pvt_thread_cond_t mdb_tool_index_cond_work; static void * mdb_tool_index_task( void *ctx, void *ptr ); -#endif static int mdb_writes, mdb_writes_per_commit; @@ -100,29 +97,30 @@ int mdb_tool_entry_open( else mdb_writes_per_commit = 1; -#if 0 /* Set up for threaded slapindex */ if (( slapMode & (SLAP_TOOL_QUICK|SLAP_TOOL_READONLY)) == SLAP_TOOL_QUICK ) { if ( !mdb_tool_info ) { + struct mdb_info *mdb = (struct mdb_info *) be->be_private; ldap_pvt_thread_mutex_init( &mdb_tool_index_mutex ); ldap_pvt_thread_cond_init( &mdb_tool_index_cond_main ); ldap_pvt_thread_cond_init( &mdb_tool_index_cond_work ); - if ( mdb->bi_nattrs ) { + if ( mdb->mi_nattrs ) { int i; - mdb_tool_index_threads = ch_malloc( slap_tool_thread_max * sizeof( int )); - mdb_tool_index_rec = ch_malloc( mdb->bi_nattrs * sizeof( IndexRec )); - mdb_tool_index_tcount = slap_tool_thread_max - 1; - for (i=1; i 1 ) { + mdb_tool_index_rec = ch_calloc( mdb->mi_nattrs, sizeof( IndexRec )); + mdb_tool_index_tcount = mdb_tool_threads - 1; + for (i=1; i 0 ) { ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main, &mdb_tool_index_mutex ); } - mdb_tool_index_tcount = slap_tool_thread_max - 1; + mdb_tool_index_tcount = mdb_tool_threads - 1; ldap_pvt_thread_cond_broadcast( &mdb_tool_index_cond_work ); /* Make sure all threads are stopped */ - while ( mdb_tool_index_tcount ) { + while ( mdb_tool_index_tcount > 0 ) { ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main, &mdb_tool_index_mutex ); } @@ -153,11 +150,9 @@ int mdb_tool_entry_close( mdb_tool_info = NULL; slapd_shutdown = 0; - ch_free( mdb_tool_index_threads ); ch_free( mdb_tool_index_rec ); - mdb_tool_index_tcount = slap_tool_thread_max - 1; + mdb_tool_index_tcount = mdb_tool_threads - 1; } -#endif if( idcursor ) { mdb_cursor_close( idcursor ); @@ -167,8 +162,8 @@ int mdb_tool_entry_close( mdb_cursor_close( cursor ); cursor = NULL; } - MDB_TOOL_IDL_FLUSH( be, txn ); if( txn ) { + MDB_TOOL_IDL_FLUSH( be, txn ); if ( mdb_txn_commit( txn )) return -1; txn = NULL; @@ -296,7 +291,7 @@ ID mdb_tool_dn2id_get( op.o_tmpmemctx = NULL; op.o_tmpmfuncs = &ch_mfuncs; - rc = mdb_dn2id( &op, txn, dn, &id, NULL, NULL ); + rc = mdb_dn2id( &op, txn, NULL, dn, &id, NULL, NULL ); if ( rc == MDB_NOTFOUND ) return NOID; @@ -398,7 +393,7 @@ static int mdb_tool_next_id( return 0; } - rc = mdb_dn2id( op, tid, &ndn, &id, NULL, &nmatched ); + rc = mdb_dn2id( op, tid, mcp, &ndn, &id, NULL, &nmatched ); if ( rc == MDB_NOTFOUND ) { if ( !be_issuffix( op->o_bd, &ndn ) ) { ID eid = e->e_id; @@ -424,7 +419,7 @@ static int mdb_tool_next_id( pid = id; } } - rc = mdb_next_id( op->o_bd, tid, &e->e_id ); + rc = mdb_next_id( op->o_bd, idcursor, &e->e_id ); if ( rc ) { snprintf( text->bv_val, text->bv_len, "next_id failed: %s (%d)", @@ -433,7 +428,7 @@ static int mdb_tool_next_id( "=> mdb_tool_next_id: %s\n", text->bv_val, 0, 0 ); return rc; } - rc = mdb_dn2id_add( op, tid, pid, e ); + rc = mdb_dn2id_add( op, mcp, mcd, pid, e ); if ( rc ) { snprintf( text->bv_val, text->bv_len, "dn2id_add failed: %s (%d)", @@ -457,7 +452,7 @@ static int mdb_tool_next_id( key.mv_data = &e->e_id; data.mv_size = 0; data.mv_data = NULL; - rc = mdb_put( tid, mdb->mi_id2entry, &key, &data, MDB_NOOVERWRITE ); + rc = mdb_cursor_put( idcursor, &key, &data, MDB_NOOVERWRITE ); if ( rc == MDB_KEYEXIST ) rc = 0; if ( rc ) { @@ -499,14 +494,14 @@ mdb_tool_index_add( if ( !mdb->mi_nattrs ) return 0; -#if 0 - if ( slapMode & SLAP_TOOL_QUICK ) { + if ( mdb_tool_threads > 1 ) { IndexRec *ir; int i, rc; Attribute *a; ir = mdb_tool_index_rec; - memset(ir, 0, mdb->bi_nattrs * sizeof( IndexRec )); + for (i=0; imi_nattrs; i++) + ir[i].ir_attrs = NULL; for ( a = e->e_attrs; a != NULL; a = a->a_next ) { rc = mdb_index_recset( mdb, a, a->a_desc->ad_type, @@ -514,39 +509,49 @@ mdb_tool_index_add( if ( rc ) return rc; } + for (i=0; imi_nattrs; i++) { + if ( !ir[i].ir_ai ) + break; + rc = mdb_cursor_open( txn, ir[i].ir_ai->ai_dbi, + &ir[i].ir_ai->ai_cursor ); + if ( rc ) + return rc; + } mdb_tool_ix_id = e->e_id; mdb_tool_ix_op = op; + mdb_tool_ix_txn = txn; ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex ); /* Wait for all threads to be ready */ while ( mdb_tool_index_tcount ) { ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main, &mdb_tool_index_mutex ); } - for ( i=1; ie_id, 0 ); + ldap_pvt_thread_cond_broadcast( &mdb_tool_index_cond_work ); + + rc = mdb_index_recrun( op, txn, mdb, ir, e->e_id, 0 ); if ( rc ) return rc; ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex ); - for ( i=1; ibe_private; if ( !txn ) { - rc = mdb_txn_begin( mdb->mi_dbenv, NULL, 0, &txn ); - if( rc != 0 ) { - snprintf( text->bv_val, text->bv_len, - "txn_begin failed: %s (%d)", - mdb_strerror(rc), rc ); - Debug( LDAP_DEBUG_ANY, - "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n", - text->bv_val, 0, 0 ); - return NOID; - } + rc = mdb_txn_begin( mdb->mi_dbenv, NULL, 0, &txn ); + if( rc != 0 ) { + snprintf( text->bv_val, text->bv_len, + "txn_begin failed: %s (%d)", + mdb_strerror(rc), rc ); + Debug( LDAP_DEBUG_ANY, + "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n", + text->bv_val, 0, 0 ); + return NOID; + } + rc = mdb_cursor_open( txn, mdb->mi_id2entry, &idcursor ); + if( rc != 0 ) { + snprintf( text->bv_val, text->bv_len, + "cursor_open failed: %s (%d)", + mdb_strerror(rc), rc ); + Debug( LDAP_DEBUG_ANY, + "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n", + text->bv_val, 0, 0 ); + return NOID; + } + rc = mdb_cursor_open( txn, mdb->mi_dn2id, &mcp ); + if( rc != 0 ) { + snprintf( text->bv_val, text->bv_len, + "cursor_open failed: %s (%d)", + mdb_strerror(rc), rc ); + Debug( LDAP_DEBUG_ANY, + "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n", + text->bv_val, 0, 0 ); + return NOID; + } + rc = mdb_cursor_open( txn, mdb->mi_dn2id, &mcd ); + if( rc != 0 ) { + snprintf( text->bv_val, text->bv_len, + "cursor_open failed: %s (%d)", + mdb_strerror(rc), rc ); + Debug( LDAP_DEBUG_ANY, + "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n", + text->bv_val, 0, 0 ); + return NOID; + } } op.o_hdr = &ohdr; @@ -610,7 +645,7 @@ ID mdb_tool_entry_put( /* id2entry index */ - rc = mdb_id2entry_add( &op, txn, e ); + rc = mdb_id2entry_add( &op, txn, idcursor, e ); if( rc != 0 ) { snprintf( text->bv_val, text->bv_len, "id2entry_add failed: err=%d", rc ); @@ -624,10 +659,14 @@ done: if( rc == 0 ) { mdb_writes++; if ( mdb_writes >= mdb_writes_per_commit ) { + unsigned i; MDB_TOOL_IDL_FLUSH( be, txn ); rc = mdb_txn_commit( txn ); + for ( i=0; imi_nattrs; i++ ) + mdb->mi_attrs[i]->ai_cursor = NULL; mdb_writes = 0; txn = NULL; + idcursor = NULL; if( rc != 0 ) { snprintf( text->bv_val, text->bv_len, "txn_commit failed: %s (%d)", @@ -642,6 +681,7 @@ done: } else { mdb_txn_abort( txn ); txn = NULL; + cursor = NULL; snprintf( text->bv_val, text->bv_len, "txn_aborted! %s (%d)", rc == LDAP_OTHER ? "Internal error" : @@ -779,8 +819,11 @@ done: if( rc == 0 ) { mdb_writes++; if ( mdb_writes >= mdb_writes_per_commit ) { + unsigned i; MDB_TOOL_IDL_FLUSH( be, txi ); rc = mdb_txn_commit( txi ); + for ( i=0; imi_nattrs; i++ ) + mi->mi_attrs[i]->ai_cursor = NULL; if( rc != 0 ) { Debug( LDAP_DEBUG_ANY, "=> " LDAP_XSTRING(mdb_tool_entry_reindex) @@ -852,7 +895,7 @@ ID mdb_tool_entry_modify( op.o_tmpmfuncs = &ch_mfuncs; /* id2entry index */ - rc = mdb_id2entry_update( &op, tid, e ); + rc = mdb_id2entry_update( &op, tid, NULL, e ); if( rc != 0 ) { snprintf( text->bv_val, text->bv_len, "id2entry_update failed: err=%d", rc ); @@ -889,7 +932,6 @@ done: return e->e_id; } -#if 0 static void * mdb_tool_index_task( void *ctx, void *ptr ) { @@ -911,14 +953,13 @@ mdb_tool_index_task( void *ctx, void *ptr ) break; } ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex ); - - mdb_tool_index_threads[base] = mdb_index_recrun( mdb_tool_ix_op, + mdb_tool_index_rec[base].ir_i = mdb_index_recrun( mdb_tool_ix_op, + mdb_tool_ix_txn, mdb_tool_info, mdb_tool_index_rec, mdb_tool_ix_id, base ); } return NULL; } -#endif #ifdef MDB_TOOL_IDL_CACHING static int @@ -932,7 +973,7 @@ mdb_tool_idl_cmp( const void *v1, const void *v2 ) } static int -mdb_tool_idl_flush_one( MDB_cursor *mc, mdb_tool_idl_cache *ic ) +mdb_tool_idl_flush_one( MDB_cursor *mc, AttrInfo *ai, mdb_tool_idl_cache *ic ) { mdb_tool_idl_cache_entry *ice; MDB_val key, data[2]; @@ -1020,24 +1061,26 @@ mdb_tool_idl_flush_one( MDB_cursor *mc, mdb_tool_idl_cache *ic ) } } if ( ic->head ) { - ic->tail->next = mdb_tool_idl_free_list; - mdb_tool_idl_free_list = ic->head; + ic->tail->next = ai->ai_flist; + ai->ai_flist = ic->head; } } - ch_free( ic ); + ic->head = ai->ai_clist; + ai->ai_clist = ic; return rc; } static int -mdb_tool_idl_flush_db( MDB_txn *txn, MDB_dbi dbi, Avlnode *root ) +mdb_tool_idl_flush_db( MDB_txn *txn, AttrInfo *ai ) { MDB_cursor *mc; + Avlnode *root; int rc; - mdb_cursor_open( txn, dbi, &mc ); - root = tavl_end( root, TAVL_DIR_LEFT ); + mdb_cursor_open( txn, ai->ai_dbi, &mc ); + root = tavl_end( ai->ai_root, TAVL_DIR_LEFT ); do { - rc = mdb_tool_idl_flush_one( mc, root->avl_data ); + rc = mdb_tool_idl_flush_one( mc, ai, root->avl_data ); if ( rc != -1 ) rc = 0; } while ((root = tavl_next(root, TAVL_DIR_RIGHT))); @@ -1054,11 +1097,10 @@ mdb_tool_idl_flush( BackendDB *be, MDB_txn *txn ) unsigned int i, dbi; for ( i=0; i < mdb->mi_nattrs; i++ ) { - dbi = mdb->mi_attrs[i]->ai_dbi; - if ( !mdb_tool_roots[dbi] ) continue; - rc = mdb_tool_idl_flush_db( txn, dbi, mdb_tool_roots[dbi] ); - tavl_free(mdb_tool_roots[dbi], NULL); - mdb_tool_roots[dbi] = NULL; + if ( !mdb->mi_attrs[i]->ai_root ) continue; + rc = mdb_tool_idl_flush_db( txn, mdb->mi_attrs[i] ); + tavl_free(mdb->mi_attrs[i]->ai_root, NULL); + mdb->mi_attrs[i]->ai_root = NULL; if ( rc ) break; } @@ -1074,11 +1116,13 @@ int mdb_tool_idl_add( mdb_tool_idl_cache *ic, itmp; mdb_tool_idl_cache_entry *ice; int i, rc, lcount; + AttrInfo *ai = (AttrInfo *)mc; + mc = ai->ai_cursor; - dbi = mdb_cursor_dbi(mc); + dbi = ai->ai_dbi; for (i=0; keys[i].bv_val; i++) { itmp.kstr = keys[i]; - ic = tavl_find( (Avlnode *)mdb_tool_roots[dbi], &itmp, mdb_tool_idl_cmp ); + ic = tavl_find( (Avlnode *)ai->ai_root, &itmp, mdb_tool_idl_cmp ); /* No entry yet, create one */ if ( !ic ) { @@ -1086,7 +1130,12 @@ int mdb_tool_idl_add( ID nid; int rc; - ic = ch_malloc( sizeof( mdb_tool_idl_cache ) + itmp.kstr.bv_len ); + if ( ai->ai_clist ) { + ic = ai->ai_clist; + ai->ai_clist = ic->head; + } else { + ic = ch_malloc( sizeof( mdb_tool_idl_cache ) + itmp.kstr.bv_len + 4 ); + } ic->kstr.bv_len = itmp.kstr.bv_len; ic->kstr.bv_val = (char *)(ic+1); memcpy( ic->kstr.bv_val, itmp.kstr.bv_val, ic->kstr.bv_len ); @@ -1095,7 +1144,7 @@ int mdb_tool_idl_add( ic->count = 0; ic->offset = 0; ic->flags = 0; - tavl_insert( (Avlnode **)&mdb_tool_roots[dbi], ic, mdb_tool_idl_cmp, + tavl_insert( (Avlnode **)&ai->ai_root, ic, mdb_tool_idl_cmp, avl_dup_error ); /* load existing key count here */ @@ -1124,12 +1173,9 @@ int mdb_tool_idl_add( continue; /* Are we at the limit, and converting to a range? */ } else if ( ic->count == MDB_IDL_DB_SIZE ) { - int n; - for ( ice = ic->head, n=0; ice; ice = ice->next, n++ ) - /* counting */ ; - if ( n ) { - ic->tail->next = mdb_tool_idl_free_list; - mdb_tool_idl_free_list = ic->head; + if ( ic->head ) { + ic->tail->next = ai->ai_flist; + ai->ai_flist = ic->head; } ic->head = ic->tail = NULL; ic->last = id; @@ -1139,12 +1185,10 @@ int mdb_tool_idl_add( /* No free block, create that too */ lcount = ic->count & (IDBLOCK-1); if ( !ic->tail || lcount == 0) { - ice = NULL; - if ( mdb_tool_idl_free_list ) { - ice = mdb_tool_idl_free_list; - mdb_tool_idl_free_list = ice->next; - } - if ( !ice ) { + if ( ai->ai_flist ) { + ice = ai->ai_flist; + ai->ai_flist = ice->next; + } else { ice = ch_malloc( sizeof( mdb_tool_idl_cache_entry )); } ice->next = NULL; -- 2.39.5