From 3e98addbe294c69b94545141f466836bf92693ee Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Sat, 5 Jul 2014 07:49:10 -0700 Subject: [PATCH 1/1] Rationalize mdb_env_copy2 API --- libraries/liblmdb/Makefile | 2 +- libraries/liblmdb/lmdb.h | 33 +++++++++++++++++------- libraries/liblmdb/mdb.c | 50 ++++++++++++++++++++++++------------ libraries/liblmdb/mdb_copy.c | 19 +++++--------- 4 files changed, 64 insertions(+), 40 deletions(-) diff --git a/libraries/liblmdb/Makefile b/libraries/liblmdb/Makefile index 60ab528f65..25c1095466 100644 --- a/libraries/liblmdb/Makefile +++ b/libraries/liblmdb/Makefile @@ -17,7 +17,7 @@ # read mdb.c before changing any of them. # CC = gcc -W = -W -Wall -Wno-unused-parameter -Wbad-function-cast +W = -W -Wall -Wno-unused-parameter -Wbad-function-cast -Wuninitialized THREADS = -pthread OPT = -O2 -g CFLAGS = $(THREADS) $(OPT) $(W) $(XCFLAGS) diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h index b5791795e6..0bc97cdf40 100644 --- a/libraries/liblmdb/lmdb.h +++ b/libraries/liblmdb/lmdb.h @@ -333,6 +333,15 @@ typedef void (MDB_rel_func)(MDB_val *item, void *oldptr, void *newptr, void *rel #define MDB_MULTIPLE 0x80000 /* @} */ +/** @defgroup mdb_copy Copy Flags + * @{ + */ +/** Compacting copy: Omit free space from copy, and renumber all + * pages sequentially. + */ +#define MDB_CP_COMPACT 0x01 +/* @} */ + /** @brief Cursor Get operations. * * This is the set of all operations for retrieving data @@ -622,14 +631,10 @@ int mdb_env_copy(MDB_env *env, const char *path); */ int mdb_env_copyfd(MDB_env *env, mdb_filehandle_t fd); - /** @brief Copy an LMDB environment to the specified path, with compaction. + /** @brief Copy an LMDB environment to the specified path, with options. * * This function may be used to make a backup of an existing environment. - * No lockfile is created, since it gets recreated at need. Unlike - * #mdb_env_copy(), which copies all pages from the environment, this - * function trims freed/unused pages from the copy and reorders leaf - * pages in sequential order. This function may execute more slowly - * than #mdb_env_copy() and will use more CPU time. + * No lockfile is created, since it gets recreated at need. * @note This call can trigger significant file size growth if run in * parallel with write transactions, because it employs a read-only * transaction. See long-lived transactions under @ref caveats_sec. @@ -638,12 +643,20 @@ int mdb_env_copyfd(MDB_env *env, mdb_filehandle_t fd); * @param[in] path The directory in which the copy will reside. This * directory must already exist and be writable but must otherwise be * empty. + * @param[in] flags Special options for this operation. This parameter + * must be set to 0 or by bitwise OR'ing together one or more of the + * values described here. + * * @return A non-zero error value on failure and 0 on success. */ -int mdb_env_copy2(MDB_env *env, const char *path); +int mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags); /** @brief Copy an LMDB environment to the specified file descriptor, - * with compaction. + * with options. * * This function may be used to make a backup of an existing environment. * No lockfile is created, since it gets recreated at need. See @@ -655,9 +668,11 @@ int mdb_env_copy2(MDB_env *env, const char *path); * must have already been opened successfully. * @param[in] fd The filedescriptor to write the copy to. It must * have already been opened for Write access. + * @param[in] flags Special options for this operation. + * See #mdb_env_copy2() for options. * @return A non-zero error value on failure and 0 on success. */ -int mdb_env_copyfd2(MDB_env *env, mdb_filehandle_t fd); +int mdb_env_copyfd2(MDB_env *env, mdb_filehandle_t fd, unsigned int flags); /** @brief Return statistics about the LMDB environment. * diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index 609eb9230a..5acdba4dac 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -8036,6 +8036,7 @@ mdb_put(MDB_txn *txn, MDB_dbi dbi, #define MDB_WBUF (1024*1024) #endif + /** State needed for a compacting copy. */ typedef struct mdb_copy { pthread_mutex_t mc_mutex; pthread_cond_t mc_cond; @@ -8050,8 +8051,10 @@ typedef struct mdb_copy { int mc_status; volatile int mc_new; int mc_toggle; + } mdb_copy; + /** Dedicated writer thread for compacting copy. */ static THREAD_RET mdb_env_copythr(void *arg) { @@ -8116,6 +8119,7 @@ again: #undef DO_WRITE } + /** Tell the writer thread there's a buffer ready to write */ static int mdb_env_cthr_toggle(mdb_copy *my, int st) { @@ -8134,6 +8138,7 @@ mdb_env_cthr_toggle(mdb_copy *my, int st) return 0; } + /** Depth-first tree traversal for compacting copy. */ static int mdb_env_cwalk(mdb_copy *my, pgno_t *pg, int flags) { @@ -8255,6 +8260,9 @@ again: mc.mc_snum++; mc.mc_ki[mc.mc_top] = 0; if (IS_BRANCH(mp)) { + /* Whenever we advance to a sibling branch page, + * we must proceed all the way down to its first leaf. + */ mdb_page_copy(mc.mc_pg[mc.mc_top], mp, my->mc_env->me_psize); goto again; } else @@ -8288,8 +8296,9 @@ done: return rc; } -int -mdb_env_copyfd2(MDB_env *env, HANDLE fd) + /** Copy environment with compaction. */ +static int +mdb_env_copyfd1(MDB_env *env, HANDLE fd) { MDB_meta *mm; MDB_page *mp; @@ -8408,8 +8417,9 @@ leave: return rc; } -int -mdb_env_copyfd(MDB_env *env, HANDLE fd) + /** Copy environment as-is. */ +static int +mdb_env_copyfd0(MDB_env *env, HANDLE fd) { MDB_txn *txn = NULL; int rc; @@ -8512,8 +8522,23 @@ leave: return rc; } -static int -mdb_env_copy0(MDB_env *env, const char *path, int flag) +int +mdb_env_copyfd2(MDB_env *env, HANDLE fd, unsigned int flags) +{ + if (flags & MDB_CP_COMPACT) + return mdb_env_copyfd1(env, fd); + else + return mdb_env_copyfd0(env, fd); +} + +int +mdb_env_copyfd(MDB_env *env, HANDLE fd) +{ + return mdb_env_copyfd2(env, fd, 0); +} + +int +mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags) { int rc, len; char *lpath; @@ -8558,10 +8583,7 @@ mdb_env_copy0(MDB_env *env, const char *path, int flag) } #endif - if (flag) - rc = mdb_env_copyfd2(env, newfd); - else - rc = mdb_env_copyfd(env, newfd); + rc = mdb_env_copyfd2(env, newfd, flags); leave: if (!(env->me_flags & MDB_NOSUBDIR)) @@ -8576,13 +8598,7 @@ leave: int mdb_env_copy(MDB_env *env, const char *path) { - return mdb_env_copy0(env, path, 0); -} - -int -mdb_env_copy2(MDB_env *env, const char *path) -{ - return mdb_env_copy0(env, path, 1); + return mdb_env_copy2(env, path, 0); } int diff --git a/libraries/liblmdb/mdb_copy.c b/libraries/liblmdb/mdb_copy.c index 0814519d4d..c54fefe9f1 100644 --- a/libraries/liblmdb/mdb_copy.c +++ b/libraries/liblmdb/mdb_copy.c @@ -33,13 +33,13 @@ int main(int argc,char * argv[]) MDB_env *env; const char *progname = argv[0], *act; unsigned flags = MDB_RDONLY; - int compact = 0; + unsigned cpflags = 0; for (; argc > 1 && argv[1][0] == '-'; argc--, argv++) { if (argv[1][1] == 'n' && argv[1][2] == '\0') flags |= MDB_NOSUBDIR; else if (argv[1][1] == 'c' && argv[1][2] == '\0') - compact = 1; + cpflags |= MDB_CP_COMPACT; else if (argv[1][1] == 'V' && argv[1][2] == '\0') { printf("%s\n", MDB_VERSION_STRING); exit(0); @@ -68,17 +68,10 @@ int main(int argc,char * argv[]) } if (rc == MDB_SUCCESS) { act = "copying"; - if (compact) { - if (argc == 2) - rc = mdb_env_copyfd2(env, MDB_STDOUT); - else - rc = mdb_env_copy2(env, argv[2]); - } else { - if (argc == 2) - rc = mdb_env_copyfd(env, MDB_STDOUT); - else - rc = mdb_env_copy(env, argv[2]); - } + if (argc == 2) + rc = mdb_env_copyfd2(env, MDB_STDOUT, cpflags); + else + rc = mdb_env_copy2(env, argv[2], cpflags); } if (rc) fprintf(stderr, "%s: %s failed, error %d (%s)\n", -- 2.39.2