From: Hallvard Furuseth Date: Thu, 7 Nov 2013 22:41:30 +0000 (+0100) Subject: Add MDB_CLEANMEM option. X-Git-Tag: OPENLDAP_REL_ENG_2_4_38~8^2~2 X-Git-Url: https://git.sur5r.net/?a=commitdiff_plain;h=92f71f50f9f2a9e93c7d757538688376a6f6debb;p=openldap Add MDB_CLEANMEM option. Initialize unused portions of DB pages, for security or to silence checkers like Purify and Valgrind. Like BDB configure --enable-umrw. --- diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h index 820dbc2f61..98c4d4548d 100644 --- a/libraries/liblmdb/lmdb.h +++ b/libraries/liblmdb/lmdb.h @@ -70,6 +70,12 @@ * access to locks and lock file. Exceptions: On read-only filesystems * or with the #MDB_NOLOCK flag described under #mdb_env_open(). * + * - By default, unused portions of the datafile may receive garbage data + * from memory freed by other code. (This does not happen when using + * the #MDB_WRITEMAP flag.) Applications handling sensitive data + * which must not be written, and which don't use #MDB_WRITEMAP, + * need to prevent this with the #MDB_CLEANMEM flag. + * * - A thread can only use one transaction at a time, plus any child * transactions. Each transaction belongs to one thread. See below. * The #MDB_NOTLS flag changes this for read-only transactions. @@ -277,6 +283,8 @@ typedef void (MDB_rel_func)(MDB_val *item, void *oldptr, void *newptr, void *rel #define MDB_NOLOCK 0x400000 /** don't do readahead (no effect on Windows) */ #define MDB_NORDAHEAD 0x800000 + /** don't write uninitialized malloc'd memory to datafile */ +#define MDB_CLEANMEM 0x1000000 /** @} */ /** @defgroup mdb_dbi_open Database Flags @@ -546,6 +554,22 @@ int mdb_env_create(MDB_env **env); * supports it. Turning it off may help random read performance * when the DB is larger than RAM and system RAM is full. * The option is not implemented on Windows. + *
  • #MDB_CLEANMEM + * Don't write uninitialized memory to unused spaces in the datafile. + * By default, memory for pages written to the datafile is obtained + * using malloc, and only the portions that LMDB uses are modified. + * Unused portions of a page may contain leftover data from other + * code that used the heap and subsequently freed that memory. + * That can be a problem for applications which handle sensitive data + * like passwords, and it makes memory checkers like Valgrind noisy. + * With this flag, unused portions of pages will be initialized to + * zero. This flag is not needed with #MDB_WRITEMAP, which writes + * directly to the mmap instead of using malloc for pages. The + * initialization is also skipped if #MDB_RESERVE is used; the + * caller is expected to overwrite all of the memory that was + * reserved in that case. + * This flag may be changed at any time using #mdb_env_set_flags(). + * It comes at some performance cost. * * @param[in] mode The UNIX permissions to set on created files. This parameter * is ignored on Windows. @@ -1131,6 +1155,8 @@ int mdb_get(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data); * reserved space, which the caller can fill in later - before * the next update operation or the transaction ends. This saves * an extra memcpy if the data is being generated later. + * MDB does nothing else with this memory, even if #MDB_CLEANMEM is + * set - the caller is expected to modify all of the space requested. *
  • #MDB_APPEND - append the given key/data pair to the end of the * database. No key comparisons are performed. This option allows * fast bulk loading when keys are already known to be in the diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index 484ab2ce04..2ef3fe4802 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -1322,7 +1322,12 @@ mdb_page_malloc(MDB_txn *txn, unsigned num) { MDB_env *env = txn->mt_env; MDB_page *ret = env->me_dpages; - size_t sz = env->me_psize; + size_t psize = env->me_psize, sz = psize, off; + /* For #MDB_CLEANMEM, psize counts how much to init. + * For a single page alloc, we init everything after the page header. + * For multi-page, we init the final page; if the caller needed that + * many pages they will be filling in at least up to the last page. + */ if (num == 1) { if (ret) { VGMEMP_ALLOC(env, ret, sz); @@ -1330,10 +1335,16 @@ mdb_page_malloc(MDB_txn *txn, unsigned num) env->me_dpages = ret->mp_next; return ret; } + psize -= off = PAGEHDRSZ; } else { sz *= num; + off = sz - psize; } if ((ret = malloc(sz)) != NULL) { + if (env->me_flags & MDB_CLEANMEM) { + memset((char *)ret + off, 0, psize); + ret->mp_pad = 0; + } VGMEMP_ALLOC(env, ret, sz); } return ret; @@ -2486,7 +2497,7 @@ mdb_freelist_save(MDB_txn *txn) int rc, maxfree_1pg = env->me_maxfree_1pg, more = 1; txnid_t pglast = 0, head_id = 0; pgno_t freecnt = 0, *free_pgs, *mop; - ssize_t head_room = 0, total_room = 0, mop_len; + ssize_t head_room = 0, total_room = 0, mop_len, clean_limit; mdb_cursor_init(&mc, txn, FREE_DBI, NULL); @@ -2497,9 +2508,15 @@ mdb_freelist_save(MDB_txn *txn) return rc; } + /* MDB_RESERVE cancels CLEANMEM in ovpage malloc (when no WRITEMAP) */ + clean_limit = (env->me_flags & (MDB_CLEANMEM|MDB_WRITEMAP)) == MDB_CLEANMEM + ? maxfree_1pg : SSIZE_MAX; + for (;;) { /* Come back here after each Put() in case freelist changed */ MDB_val key, data; + pgno_t *pgs; + ssize_t j; /* If using records from freeDB which we have not yet * deleted, delete them and any we reserved for me_pghead. @@ -2583,7 +2600,12 @@ mdb_freelist_save(MDB_txn *txn) rc = mdb_cursor_put(&mc, &key, &data, MDB_RESERVE); if (rc) return rc; - *(MDB_ID *)data.mv_data = 0; /* IDL is initially empty */ + /* IDL is initially empty, zero out at least the length */ + pgs = (pgno_t *)data.mv_data; + j = head_room > clean_limit ? head_room : 0; + do { + pgs[j] = 0; + } while (--j >= 0); total_room += head_room; } @@ -3943,8 +3965,9 @@ fail: * at runtime. Changing other flags requires closing the * environment and re-opening it with the new flags. */ -#define CHANGEABLE (MDB_NOSYNC|MDB_NOMETASYNC|MDB_MAPASYNC) -#define CHANGELESS (MDB_FIXEDMAP|MDB_NOSUBDIR|MDB_RDONLY|MDB_WRITEMAP|MDB_NOTLS|MDB_NOLOCK|MDB_NORDAHEAD) +#define CHANGEABLE (MDB_NOSYNC|MDB_NOMETASYNC|MDB_MAPASYNC|MDB_CLEANMEM) +#define CHANGELESS (MDB_FIXEDMAP|MDB_NOSUBDIR|MDB_RDONLY|MDB_WRITEMAP| \ + MDB_NOTLS|MDB_NOLOCK|MDB_NORDAHEAD) int mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode) @@ -5847,12 +5870,14 @@ more: if (NODESIZE + sizeof(indx_t) + NODEKSZ(leaf) + xdata.mv_size >= env->me_nodemax) { /* yes, convert it */ - dummy.md_flags = 0; if (mc->mc_db->md_flags & MDB_DUPFIXED) { dummy.md_pad = fp->mp_pad; dummy.md_flags = MDB_DUPFIXED; if (mc->mc_db->md_flags & MDB_INTEGERDUP) dummy.md_flags |= MDB_INTEGERKEY; + } else { + dummy.md_pad = 0; + dummy.md_flags = 0; } dummy.md_depth = 1; dummy.md_branch_pages = 0; diff --git a/libraries/liblmdb/mtest.c b/libraries/liblmdb/mtest.c index dbc69b8d4c..f9a96b2ab8 100644 --- a/libraries/liblmdb/mtest.c +++ b/libraries/liblmdb/mtest.c @@ -28,7 +28,7 @@ int main(int argc,char * argv[]) MDB_cursor *cursor, *cur2; int count; int *values; - char sval[32]; + char sval[32] = ""; srandom(time(NULL)); diff --git a/libraries/liblmdb/mtest2.c b/libraries/liblmdb/mtest2.c index 44d1de7ccd..bfa29f9f74 100644 --- a/libraries/liblmdb/mtest2.c +++ b/libraries/liblmdb/mtest2.c @@ -31,7 +31,7 @@ int main(int argc,char * argv[]) MDB_cursor *cursor; int count; int *values; - char sval[32]; + char sval[32] = ""; srandom(time(NULL));