From: Quanah Gibson-Mount Date: Mon, 10 Nov 2008 18:12:13 +0000 (+0000) Subject: dbpagesize, checksum keywords X-Git-Tag: OPENLDAP_REL_ENG_2_4_13~115 X-Git-Url: https://git.sur5r.net/?a=commitdiff_plain;h=bda59e77f163c6f972c507f2122d736d1f794c3c;p=openldap dbpagesize, checksum keywords --- diff --git a/CHANGES b/CHANGES index a5a4321631..4ad7d1ab77 100644 --- a/CHANGES +++ b/CHANGES @@ -7,6 +7,8 @@ OpenLDAP 2.4.13 Engineering Added slapd private databases in global overlays (ITS#5735,ITS#5736) Added slapd support for certificateListExactMatch (ITS#5700) Fixed slapd-bdb/hdb invalid db crash (ITS#5698) + Added slapd-bdb/hdb dbpagesize keyword + Added slapd-bdb/hdb checksum keyword Fixed slapo-chain/translucent back-config support (ITS#5736) Added slapo-constraint support for LDAP URI constraints (ITS#5704) Added slapo-constraint support for constraining rename (ITS#5703) diff --git a/doc/man/man5/slapd-bdb.5 b/doc/man/man5/slapd-bdb.5 index 6bce8f0040..25a785f173 100644 --- a/doc/man/man5/slapd-bdb.5 +++ b/doc/man/man5/slapd-bdb.5 @@ -60,6 +60,10 @@ the \fI\fP argument is non-zero, an internal task will run every \fI\fP minutes to perform the checkpoint. See the Berkeley DB reference guide for more details. .TP +.B checksum +Enable checksum validation of DB pages whenever they are read from disk. +This setting can only be configured before any database files are created. +.TP .BI cryptfile \ Specify the pathname of a file containing an encryption key to use for encrypting the database. Encryption is performed using Berkeley DB's @@ -111,6 +115,19 @@ Enabling this option may improve performance at the expense of data security. See the Berkeley DB reference guide for more details. .TP +\fBdbpagesize \fR \fI \fR +Specify the page size to use for a particular database file, in units +of 1024 bytes. The default for the +.B id2entry +file is 16, the default for all other files depends on the size of the +underlying filesystem's block size (typically 4 or 8). +The maximum that BerkeleyDB supports is 64. This +setting usually should not need to be changed, but if BerkeleyDB's +"db_stat -d" shows a large amount of overflow pages in use in a file, +setting a larger size may increase performance at the expense of +data integrity. This setting only takes effect when a database is +being newly created. See the Berkeley DB reference guide for more details. +.TP .BI directory \ Specify the directory where the BDB files containing this database and associated indexes live. diff --git a/servers/slapd/back-bdb/back-bdb.h b/servers/slapd/back-bdb/back-bdb.h index 6493a4583c..967da64768 100644 --- a/servers/slapd/back-bdb/back-bdb.h +++ b/servers/slapd/back-bdb/back-bdb.h @@ -54,10 +54,6 @@ LDAP_BEGIN_DECL #define BDB_ID2ENTRY_PAGESIZE 16384 #endif -#ifndef BDB_PAGESIZE -#define BDB_PAGESIZE 4096 /* BDB's original default */ -#endif - #define DEFAULT_CACHE_SIZE 1000 /* The default search IDL stack cache depth */ @@ -156,6 +152,12 @@ struct bdb_db_info { DB *bdi_db; }; +struct bdb_db_pgsize { + struct bdb_db_pgsize *bdp_next; + struct berval bdp_name; + int bdp_size; +}; + #ifdef LDAP_DEVEL #define BDB_MONITOR_IDX #endif /* LDAP_DEVEL */ @@ -178,9 +180,10 @@ struct bdb_info { int bi_dbenv_mode; int bi_ndatabases; + int bi_db_opflags; /* db-specific flags */ struct bdb_db_info **bi_databases; ldap_pvt_thread_mutex_t bi_database_mutex; - int bi_db_opflags; /* db-specific flags */ + struct bdb_db_pgsize *bi_pagesizes; slap_mask_t bi_defaultmask; Cache bi_cache; @@ -226,6 +229,7 @@ struct bdb_info { #define BDB_UPD_CONFIG 0x04 #define BDB_DEL_INDEX 0x08 #define BDB_RE_OPEN 0x10 +#define BDB_CHKSUM 0x20 #ifdef BDB_HIER int bi_modrdns; /* number of modrdns completed */ ldap_pvt_thread_mutex_t bi_modrdns_mutex; diff --git a/servers/slapd/back-bdb/config.c b/servers/slapd/back-bdb/config.c index 40fca1f8c3..74331beb9e 100644 --- a/servers/slapd/back-bdb/config.c +++ b/servers/slapd/back-bdb/config.c @@ -50,7 +50,9 @@ enum { BDB_INDEX, BDB_LOCKD, BDB_SSTACK, - BDB_MODE + BDB_MODE, + BDB_PGSIZE, + BDB_CHECKSUM }; static ConfigTable bdbcfg[] = { @@ -73,6 +75,10 @@ static ConfigTable bdbcfg[] = { bdb_cf_gen, "( OLcfgDbAt:1.2 NAME 'olcDbCheckpoint' " "DESC 'Database checkpoint interval in kbytes and minutes' " "SYNTAX OMsDirectoryString SINGLE-VALUE )",NULL, NULL }, + { "checksum", NULL, 1, 2, 0, ARG_ON_OFF|ARG_MAGIC|BDB_CHECKSUM, + bdb_cf_gen, "( OLcfgDbAt:1.16 NAME 'olcDbChecksum' " + "DESC 'Enable database checksum validation' " + "SYNTAX OMsBoolean SINGLE-VALUE )", NULL, NULL }, { "cryptfile", "file", 2, 2, 0, ARG_STRING|ARG_MAGIC|BDB_CRYPTFILE, bdb_cf_gen, "( OLcfgDbAt:1.13 NAME 'olcDbCryptFile' " "DESC 'Pathname of file containing the DB encryption key' " @@ -89,6 +95,11 @@ static ConfigTable bdbcfg[] = { bdb_cf_gen, "( OLcfgDbAt:1.4 NAME 'olcDbNoSync' " "DESC 'Disable synchronous database writes' " "SYNTAX OMsBoolean SINGLE-VALUE )", NULL, NULL }, + { "dbpagesize", "db> value_int = 1; break; + case BDB_CHECKSUM: + if ( bdb->bi_flags & BDB_CHKSUM ) + c->value_int = 1; + break; + case BDB_INDEX: bdb_attr_index_unparse( bdb, &c->rvalue_vals ); if ( !c->rvalue_vals ) rc = 1; @@ -483,6 +499,23 @@ bdb_cf_gen( ConfigArgs *c ) case BDB_SSTACK: c->value_int = bdb->bi_search_stack_depth; break; + + case BDB_PGSIZE: { + struct bdb_db_pgsize *ps; + char buf[SLAP_TEXT_BUFLEN]; + struct berval bv; + int rc = 1; + + bv.bv_val = buf; + for ( ps = bdb->bi_pagesizes; ps; ps = ps->bdp_next ) { + bv.bv_len = sprintf( buf, "%s %d", ps->bdp_name.bv_val, + ps->bdp_size / 1024 ); + value_add_one( &c->rvalue_vals, &bv ); + rc = 0; + + } + break; + } } return rc; } else if ( c->op == LDAP_MOD_DELETE ) { @@ -554,6 +587,9 @@ bdb_cf_gen( ConfigArgs *c ) case BDB_NOSYNC: bdb->bi_dbenv->set_flags( bdb->bi_dbenv, DB_TXN_NOSYNC, 0 ); break; + case BDB_CHECKSUM: + bdb->bi_flags &= ~BDB_CHKSUM; + break; case BDB_INDEX: if ( c->valx == -1 ) { int i; @@ -608,6 +644,24 @@ bdb_cf_gen( ConfigArgs *c ) } } break; + /* doesn't make sense on the fly; the DB file must be + * recreated + */ + case BDB_PGSIZE: { + struct bdb_db_pgsize *ps, **prev; + int i; + + for ( i = 0, prev = &bdb->bi_pagesizes, ps = *prev; ps; + prev = &ps->bdp_next, ps = ps->bdp_next, i++ ) { + if ( c->valx == -1 || i == c->valx ) { + *prev = ps->bdp_next; + ch_free( ps ); + ps = *prev; + if ( i == c->valx ) break; + } + } + } + break; } return rc; } @@ -798,6 +852,13 @@ bdb_cf_gen( ConfigArgs *c ) } break; + case BDB_CHECKSUM: + if ( c->value_int ) + bdb->bi_flags |= BDB_CHKSUM; + else + bdb->bi_flags &= ~BDB_CHKSUM; + break; + case BDB_INDEX: rc = bdb_attr_index_config( bdb, c->fname, c->lineno, c->argc - 1, &c->argv[1], &c->reply); @@ -841,6 +902,31 @@ bdb_cf_gen( ConfigArgs *c ) } bdb->bi_search_stack_depth = c->value_int; break; + + case BDB_PGSIZE: { + struct bdb_db_pgsize *ps, **prev; + int i, s; + + s = atoi(c->argv[2]); + if ( s < 1 || s > 64 ) { + snprintf( c->cr_msg, sizeof( c->cr_msg ), + "%s: size must be > 0 and <= 64: %d", + c->log, s ); + Debug( LDAP_DEBUG_ANY, "%s\n", c->cr_msg, 0, 0 ); + return -1; + } + i = strlen(c->argv[1]); + ps = ch_malloc( sizeof(struct bdb_db_pgsize) + i + 1 ); + ps->bdp_next = NULL; + ps->bdp_name.bv_len = i; + ps->bdp_name.bv_val = (char *)(ps+1); + strcpy( ps->bdp_name.bv_val, c->argv[1] ); + ps->bdp_size = s * 1024; + for ( prev = &bdb->bi_pagesizes; *prev; prev = &(*prev)->bdp_next ) + ; + *prev = ps; + } + break; } return 0; } diff --git a/servers/slapd/back-bdb/dbcache.c b/servers/slapd/back-bdb/dbcache.c index c6b8b23684..e9ff471ef7 100644 --- a/servers/slapd/back-bdb/dbcache.c +++ b/servers/slapd/back-bdb/dbcache.c @@ -57,6 +57,29 @@ bdb_db_hash( #define BDB_INDEXTYPE DB_BTREE #endif +/* If a configured size is found, return it, otherwise return 0 */ +int +bdb_db_findsize( + struct bdb_info *bdb, + struct berval *name +) +{ + struct bdb_db_pgsize *bp; + int rc; + + for ( bp = bdb->bi_pagesizes; bp; bp=bp->bdp_next ) { + rc = strncmp( name->bv_val, bp->bdp_name.bv_val, name->bv_len ); + if ( !rc ) { + if ( name->bv_len == bp->bdp_name.bv_len ) + return bp->bdp_size; + if ( name->bv_len < bp->bdp_name.bv_len && + bp->bdp_name.bv_val[name->bv_len] == '.' ) + return bp->bdp_size; + } + } + return 0; +} + int bdb_db_cache( Backend *be, @@ -121,7 +144,24 @@ bdb_db_cache( } } - rc = db->bdi_db->set_pagesize( db->bdi_db, BDB_PAGESIZE ); + if( bdb->bi_flags & BDB_CHKSUM ) { + rc = db->bdi_db->set_flags( db->bdi_db, DB_CHKSUM ); + if ( rc ) { + Debug( LDAP_DEBUG_ANY, + "bdb_db_cache: db set_flags(DB_CHKSUM)(%s) failed: %s (%d)\n", + bdb->bi_dbenv_home, db_strerror(rc), rc ); + ldap_pvt_thread_mutex_unlock( &bdb->bi_database_mutex ); + db->bdi_db->close( db->bdi_db, 0 ); + ch_free( db ); + return rc; + } + } + + /* If no explicit size set, use the FS default */ + flags = bdb_db_findsize( bdb, name ); + if ( flags ) + rc = db->bdi_db->set_pagesize( db->bdi_db, flags ); + #ifdef BDB_INDEX_USE_HASH rc = db->bdi_db->set_h_hash( db->bdi_db, bdb_db_hash ); #endif diff --git a/servers/slapd/back-bdb/init.c b/servers/slapd/back-bdb/init.c index 393b6a52f4..20b5fe9a66 100644 --- a/servers/slapd/back-bdb/init.c +++ b/servers/slapd/back-bdb/init.c @@ -416,19 +416,40 @@ shm_retry: } } + if( bdb->bi_flags & BDB_CHKSUM ) { + rc = db->bdi_db->set_flags( db->bdi_db, DB_CHKSUM ); + if ( rc ) { + snprintf(cr->msg, sizeof(cr->msg), + "database \"%s\": db set_flags(DB_CHKSUM)(%s) failed: %s (%d).", + be->be_suffix[0].bv_val, + bdb->bi_dbenv_home, db_strerror(rc), rc ); + Debug( LDAP_DEBUG_ANY, + LDAP_XSTRING(bdb_db_open) ": %s\n", + cr->msg, 0, 0 ); + goto fail; + } + } + + rc = bdb_db_findsize( bdb, (struct berval *)&bdbi_databases[i].name ); + if( i == BDB_ID2ENTRY ) { + if ( !rc ) rc = BDB_ID2ENTRY_PAGESIZE; + rc = db->bdi_db->set_pagesize( db->bdi_db, rc ); + if ( slapMode & SLAP_TOOL_MODE ) db->bdi_db->mpf->set_priority( db->bdi_db->mpf, DB_PRIORITY_VERY_LOW ); - rc = db->bdi_db->set_pagesize( db->bdi_db, - BDB_ID2ENTRY_PAGESIZE ); if ( slapMode & SLAP_TOOL_READMAIN ) { flags |= DB_RDONLY; } else { flags |= DB_CREATE; } } else { + /* Use FS default size if not configured */ + if ( rc ) + rc = db->bdi_db->set_pagesize( db->bdi_db, rc ); + rc = db->bdi_db->set_flags( db->bdi_db, DB_DUP | DB_DUPSORT ); #ifndef BDB_HIER @@ -446,8 +467,6 @@ shm_retry: flags |= DB_CREATE; } #endif - rc = db->bdi_db->set_pagesize( db->bdi_db, - BDB_PAGESIZE ); } #ifdef HAVE_EBCDIC diff --git a/servers/slapd/back-bdb/proto-bdb.h b/servers/slapd/back-bdb/proto-bdb.h index 188d68e13c..0659c6b1e2 100644 --- a/servers/slapd/back-bdb/proto-bdb.h +++ b/servers/slapd/back-bdb/proto-bdb.h @@ -70,6 +70,7 @@ int bdb_back_init_cf( BackendInfo *bi ); * dbcache.c */ #define bdb_db_cache BDB_SYMBOL(db_cache) +#define bdb_db_findsize BDB_SYMBOL(db_findsize) int bdb_db_cache( @@ -77,6 +78,11 @@ bdb_db_cache( struct berval *name, DB **db ); +int +bdb_db_findsize( + struct bdb_info *bdb, + struct berval *name ); + /* * dn2entry.c */