]> git.sur5r.net Git - openldap/commitdiff
dbpagesize, checksum keywords
authorQuanah Gibson-Mount <quanah@openldap.org>
Mon, 10 Nov 2008 18:12:13 +0000 (18:12 +0000)
committerQuanah Gibson-Mount <quanah@openldap.org>
Mon, 10 Nov 2008 18:12:13 +0000 (18:12 +0000)
CHANGES
doc/man/man5/slapd-bdb.5
servers/slapd/back-bdb/back-bdb.h
servers/slapd/back-bdb/config.c
servers/slapd/back-bdb/dbcache.c
servers/slapd/back-bdb/init.c
servers/slapd/back-bdb/proto-bdb.h

diff --git a/CHANGES b/CHANGES
index a5a4321631969041b767e54a07ed45cb2d513821..4ad7d1ab77ddb9127ed5e6b2cacf1ebbc52d9713 100644 (file)
--- a/CHANGES
+++ b/CHANGES
@@ -7,6 +7,8 @@ OpenLDAP 2.4.13 Engineering
        Added slapd private databases in global overlays (ITS#5735,ITS#5736)
        Added slapd support for certificateListExactMatch (ITS#5700)
        Fixed slapd-bdb/hdb invalid db crash (ITS#5698)
+       Added slapd-bdb/hdb dbpagesize keyword
+       Added slapd-bdb/hdb checksum keyword
        Fixed slapo-chain/translucent back-config support (ITS#5736)
        Added slapo-constraint support for LDAP URI constraints (ITS#5704)
        Added slapo-constraint support for constraining rename (ITS#5703)
index 6bce8f00409d9883913f77a278b91ab32a134eac..25a785f17347b03d76df86cd24b327d15c2336ee 100644 (file)
@@ -60,6 +60,10 @@ the \fI<min>\fP argument is non-zero, an internal task will run every
 \fI<min>\fP minutes to perform the checkpoint.
 See the Berkeley DB reference guide for more details.
 .TP
+.B checksum
+Enable checksum validation of DB pages whenever they are read from disk.
+This setting can only be configured before any database files are created.
+.TP
 .BI cryptfile \ <file>
 Specify the pathname of a file containing an encryption key to use for
 encrypting the database. Encryption is performed using Berkeley DB's
@@ -111,6 +115,19 @@ Enabling this option may improve performance at the expense of data
 security.
 See the Berkeley DB reference guide for more details.
 .TP
+\fBdbpagesize \fR \fI<dbfile> <size>\fR
+Specify the page size to use for a particular database file, in units
+of 1024 bytes. The default for the
+.B id2entry
+file is 16, the default for all other files depends on the size of the
+underlying filesystem's block size (typically 4 or 8).
+The maximum that BerkeleyDB supports is 64. This
+setting usually should not need to be changed, but if BerkeleyDB's
+"db_stat -d" shows a large amount of overflow pages in use in a file,
+setting a larger size may increase performance at the expense of
+data integrity. This setting only takes effect when a database is
+being newly created. See the Berkeley DB reference guide for more details.
+.TP
 .BI directory \ <directory>
 Specify the directory where the BDB files containing this database and
 associated indexes live.
index 6493a4583c96d75d380fb6cd3cd25fe4a0074035..967da64768f6ad7485c348e331c881f8a92e8864 100644 (file)
@@ -54,10 +54,6 @@ LDAP_BEGIN_DECL
 #define        BDB_ID2ENTRY_PAGESIZE   16384
 #endif
 
-#ifndef BDB_PAGESIZE
-#define        BDB_PAGESIZE    4096    /* BDB's original default */
-#endif
-
 #define DEFAULT_CACHE_SIZE     1000
 
 /* The default search IDL stack cache depth */
@@ -156,6 +152,12 @@ struct bdb_db_info {
        DB                      *bdi_db;
 };
 
+struct bdb_db_pgsize {
+       struct bdb_db_pgsize *bdp_next;
+       struct berval   bdp_name;
+       int     bdp_size;
+};
+
 #ifdef LDAP_DEVEL
 #define BDB_MONITOR_IDX
 #endif /* LDAP_DEVEL */
@@ -178,9 +180,10 @@ struct bdb_info {
        int                     bi_dbenv_mode;
 
        int                     bi_ndatabases;
+       int             bi_db_opflags;  /* db-specific flags */
        struct bdb_db_info **bi_databases;
        ldap_pvt_thread_mutex_t bi_database_mutex;
-       int             bi_db_opflags;  /* db-specific flags */
+       struct bdb_db_pgsize *bi_pagesizes;
 
        slap_mask_t     bi_defaultmask;
        Cache           bi_cache;
@@ -226,6 +229,7 @@ struct bdb_info {
 #define        BDB_UPD_CONFIG  0x04
 #define        BDB_DEL_INDEX   0x08
 #define        BDB_RE_OPEN             0x10
+#define BDB_CHKSUM             0x20
 #ifdef BDB_HIER
        int             bi_modrdns;             /* number of modrdns completed */
        ldap_pvt_thread_mutex_t bi_modrdns_mutex;
index 40fca1f8c321130fad49ad7b467f3747e6cdeec9..74331beb9ed6ee6a0357128835848fb32f892284 100644 (file)
@@ -50,7 +50,9 @@ enum {
        BDB_INDEX,
        BDB_LOCKD,
        BDB_SSTACK,
-       BDB_MODE
+       BDB_MODE,
+       BDB_PGSIZE,
+       BDB_CHECKSUM
 };
 
 static ConfigTable bdbcfg[] = {
@@ -73,6 +75,10 @@ static ConfigTable bdbcfg[] = {
                bdb_cf_gen, "( OLcfgDbAt:1.2 NAME 'olcDbCheckpoint' "
                        "DESC 'Database checkpoint interval in kbytes and minutes' "
                        "SYNTAX OMsDirectoryString SINGLE-VALUE )",NULL, NULL },
+       { "checksum", NULL, 1, 2, 0, ARG_ON_OFF|ARG_MAGIC|BDB_CHECKSUM,
+               bdb_cf_gen, "( OLcfgDbAt:1.16 NAME 'olcDbChecksum' "
+                       "DESC 'Enable database checksum validation' "
+                       "SYNTAX OMsBoolean SINGLE-VALUE )", NULL, NULL },
        { "cryptfile", "file", 2, 2, 0, ARG_STRING|ARG_MAGIC|BDB_CRYPTFILE,
                bdb_cf_gen, "( OLcfgDbAt:1.13 NAME 'olcDbCryptFile' "
                        "DESC 'Pathname of file containing the DB encryption key' "
@@ -89,6 +95,11 @@ static ConfigTable bdbcfg[] = {
                bdb_cf_gen, "( OLcfgDbAt:1.4 NAME 'olcDbNoSync' "
                        "DESC 'Disable synchronous database writes' "
                        "SYNTAX OMsBoolean SINGLE-VALUE )", NULL, NULL },
+       { "dbpagesize", "db> <size", 3, 3, 0, ARG_MAGIC|BDB_PGSIZE,
+               bdb_cf_gen, "( OLcfgDbAt:1.15 NAME 'olcDbPageSize' "
+                       "DESC 'Page size of specified DB, in Kbytes' "
+                       "EQUALITY caseExactMatch "
+                       "SYNTAX OMsDirectoryString )", NULL, NULL },
        { "dirtyread", NULL, 1, 2, 0,
 #ifdef SLAP_BDB_ALLOW_DIRTY_READ
                ARG_ON_OFF|ARG_MAGIC|BDB_DIRTYR, bdb_cf_gen,
@@ -157,7 +168,7 @@ static ConfigOCs bdbocs[] = {
                "olcDbNoSync $ olcDbDirtyRead $ olcDbIDLcacheSize $ "
                "olcDbIndex $ olcDbLinearIndex $ olcDbLockDetect $ "
                "olcDbMode $ olcDbSearchStack $ olcDbShmKey $ "
-               "olcDbCacheFree $ olcDbDNcacheSize ) )",
+               "olcDbCacheFree $ olcDbDNcacheSize $ olcDbPageSize ) )",
                        Cft_Database, bdbcfg },
        { NULL, 0, NULL }
 };
@@ -461,6 +472,11 @@ bdb_cf_gen( ConfigArgs *c )
                                c->value_int = 1;
                        break;
                        
+               case BDB_CHECKSUM:
+                       if ( bdb->bi_flags & BDB_CHKSUM )
+                               c->value_int = 1;
+                       break;
+
                case BDB_INDEX:
                        bdb_attr_index_unparse( bdb, &c->rvalue_vals );
                        if ( !c->rvalue_vals ) rc = 1;
@@ -483,6 +499,23 @@ bdb_cf_gen( ConfigArgs *c )
                case BDB_SSTACK:
                        c->value_int = bdb->bi_search_stack_depth;
                        break;
+
+               case BDB_PGSIZE: {
+                               struct bdb_db_pgsize *ps;
+                               char buf[SLAP_TEXT_BUFLEN];
+                               struct berval bv;
+                               int rc = 1;
+
+                               bv.bv_val = buf;
+                               for ( ps = bdb->bi_pagesizes; ps; ps = ps->bdp_next ) {
+                                       bv.bv_len = sprintf( buf, "%s %d", ps->bdp_name.bv_val,
+                                               ps->bdp_size / 1024 );
+                                       value_add_one( &c->rvalue_vals, &bv );
+                                       rc = 0;
+
+                               }
+                               break;
+                       }
                }
                return rc;
        } else if ( c->op == LDAP_MOD_DELETE ) {
@@ -554,6 +587,9 @@ bdb_cf_gen( ConfigArgs *c )
                case BDB_NOSYNC:
                        bdb->bi_dbenv->set_flags( bdb->bi_dbenv, DB_TXN_NOSYNC, 0 );
                        break;
+               case BDB_CHECKSUM:
+                       bdb->bi_flags &= ~BDB_CHKSUM;
+                       break;
                case BDB_INDEX:
                        if ( c->valx == -1 ) {
                                int i;
@@ -608,6 +644,24 @@ bdb_cf_gen( ConfigArgs *c )
                                }
                        }
                        break;
+               /* doesn't make sense on the fly; the DB file must be
+                * recreated
+                */
+               case BDB_PGSIZE: {
+                               struct bdb_db_pgsize *ps, **prev;
+                               int i;
+
+                               for ( i = 0, prev = &bdb->bi_pagesizes, ps = *prev; ps;
+                                       prev = &ps->bdp_next, ps = ps->bdp_next, i++ ) {
+                                       if ( c->valx == -1 || i == c->valx ) {
+                                               *prev = ps->bdp_next;
+                                               ch_free( ps );
+                                               ps = *prev;
+                                               if ( i == c->valx ) break;
+                                       }
+                               }
+                       }
+                       break;
                }
                return rc;
        }
@@ -798,6 +852,13 @@ bdb_cf_gen( ConfigArgs *c )
                }
                break;
 
+       case BDB_CHECKSUM:
+               if ( c->value_int )
+                       bdb->bi_flags |= BDB_CHKSUM;
+               else
+                       bdb->bi_flags &= ~BDB_CHKSUM;
+               break;
+
        case BDB_INDEX:
                rc = bdb_attr_index_config( bdb, c->fname, c->lineno,
                        c->argc - 1, &c->argv[1], &c->reply);
@@ -841,6 +902,31 @@ bdb_cf_gen( ConfigArgs *c )
                }
                bdb->bi_search_stack_depth = c->value_int;
                break;
+
+       case BDB_PGSIZE: {
+               struct bdb_db_pgsize *ps, **prev;
+               int i, s;
+               
+               s = atoi(c->argv[2]);
+               if ( s < 1 || s > 64 ) {
+                       snprintf( c->cr_msg, sizeof( c->cr_msg ),
+                               "%s: size must be > 0 and <= 64: %d",
+                               c->log, s );
+                       Debug( LDAP_DEBUG_ANY, "%s\n", c->cr_msg, 0, 0 );
+                       return -1;
+               }
+               i = strlen(c->argv[1]);
+               ps = ch_malloc( sizeof(struct bdb_db_pgsize) + i + 1 );
+               ps->bdp_next = NULL;
+               ps->bdp_name.bv_len = i;
+               ps->bdp_name.bv_val = (char *)(ps+1);
+               strcpy( ps->bdp_name.bv_val, c->argv[1] );
+               ps->bdp_size = s * 1024;
+               for ( prev = &bdb->bi_pagesizes; *prev; prev = &(*prev)->bdp_next )
+                       ;
+               *prev = ps;
+               }
+               break;
        }
        return 0;
 }
index c6b8b2368411553f0753309d44a4c5568baaa789..e9ff471ef7524c8e1b40ad15c13683c3a2e5505a 100644 (file)
@@ -57,6 +57,29 @@ bdb_db_hash(
 #define        BDB_INDEXTYPE   DB_BTREE
 #endif
 
+/* If a configured size is found, return it, otherwise return 0 */
+int
+bdb_db_findsize(
+       struct bdb_info *bdb,
+       struct berval *name
+)
+{
+       struct bdb_db_pgsize *bp;
+       int rc;
+
+       for ( bp = bdb->bi_pagesizes; bp; bp=bp->bdp_next ) {
+               rc = strncmp( name->bv_val, bp->bdp_name.bv_val, name->bv_len );
+               if ( !rc ) {
+                       if ( name->bv_len == bp->bdp_name.bv_len )
+                               return bp->bdp_size;
+                       if ( name->bv_len < bp->bdp_name.bv_len &&
+                               bp->bdp_name.bv_val[name->bv_len] == '.' )
+                               return bp->bdp_size;
+               }
+       }
+       return 0;
+}
+
 int
 bdb_db_cache(
        Backend *be,
@@ -121,7 +144,24 @@ bdb_db_cache(
                }
        }
 
-       rc = db->bdi_db->set_pagesize( db->bdi_db, BDB_PAGESIZE );
+       if( bdb->bi_flags & BDB_CHKSUM ) {
+               rc = db->bdi_db->set_flags( db->bdi_db, DB_CHKSUM );
+               if ( rc ) {
+                       Debug( LDAP_DEBUG_ANY,
+                               "bdb_db_cache: db set_flags(DB_CHKSUM)(%s) failed: %s (%d)\n",
+                               bdb->bi_dbenv_home, db_strerror(rc), rc );
+                       ldap_pvt_thread_mutex_unlock( &bdb->bi_database_mutex );
+                       db->bdi_db->close( db->bdi_db, 0 );
+                       ch_free( db );
+                       return rc;
+               }
+       }
+
+       /* If no explicit size set, use the FS default */
+       flags = bdb_db_findsize( bdb, name );
+       if ( flags )
+               rc = db->bdi_db->set_pagesize( db->bdi_db, flags );
+
 #ifdef BDB_INDEX_USE_HASH
        rc = db->bdi_db->set_h_hash( db->bdi_db, bdb_db_hash );
 #endif
index 393b6a52f42038b677814c6533ec9d866498eb22..20b5fe9a66e8bf1b9c035915de37178ed652ec49 100644 (file)
@@ -416,19 +416,40 @@ shm_retry:
                        }
                }
 
+               if( bdb->bi_flags & BDB_CHKSUM ) {
+                       rc = db->bdi_db->set_flags( db->bdi_db, DB_CHKSUM );
+                       if ( rc ) {
+                               snprintf(cr->msg, sizeof(cr->msg),
+                                       "database \"%s\": db set_flags(DB_CHKSUM)(%s) failed: %s (%d).",
+                                       be->be_suffix[0].bv_val, 
+                                       bdb->bi_dbenv_home, db_strerror(rc), rc );
+                               Debug( LDAP_DEBUG_ANY,
+                                       LDAP_XSTRING(bdb_db_open) ": %s\n",
+                                       cr->msg, 0, 0 );
+                               goto fail;
+                       }
+               }
+
+               rc = bdb_db_findsize( bdb, (struct berval *)&bdbi_databases[i].name );
+
                if( i == BDB_ID2ENTRY ) {
+                       if ( !rc ) rc = BDB_ID2ENTRY_PAGESIZE;
+                       rc = db->bdi_db->set_pagesize( db->bdi_db, rc );
+
                        if ( slapMode & SLAP_TOOL_MODE )
                                db->bdi_db->mpf->set_priority( db->bdi_db->mpf,
                                        DB_PRIORITY_VERY_LOW );
 
-                       rc = db->bdi_db->set_pagesize( db->bdi_db,
-                               BDB_ID2ENTRY_PAGESIZE );
                        if ( slapMode & SLAP_TOOL_READMAIN ) {
                                flags |= DB_RDONLY;
                        } else {
                                flags |= DB_CREATE;
                        }
                } else {
+                       /* Use FS default size if not configured */
+                       if ( rc )
+                               rc = db->bdi_db->set_pagesize( db->bdi_db, rc );
+
                        rc = db->bdi_db->set_flags( db->bdi_db, 
                                DB_DUP | DB_DUPSORT );
 #ifndef BDB_HIER
@@ -446,8 +467,6 @@ shm_retry:
                                flags |= DB_CREATE;
                        }
 #endif
-                       rc = db->bdi_db->set_pagesize( db->bdi_db,
-                               BDB_PAGESIZE );
                }
 
 #ifdef HAVE_EBCDIC
index 188d68e13c9dbfaa1bc48a65e88e775c1ac7c927..0659c6b1e261ecc8a56e59712e0a4316aa202131 100644 (file)
@@ -70,6 +70,7 @@ int bdb_back_init_cf( BackendInfo *bi );
  * dbcache.c
  */
 #define bdb_db_cache                           BDB_SYMBOL(db_cache)
+#define bdb_db_findsize                                BDB_SYMBOL(db_findsize)
 
 int
 bdb_db_cache(
@@ -77,6 +78,11 @@ bdb_db_cache(
     struct berval *name,
        DB **db );
 
+int
+bdb_db_findsize(
+       struct bdb_info *bdb,
+       struct berval *name );
+
 /*
  * dn2entry.c
  */