X-Git-Url: https://git.sur5r.net/?a=blobdiff_plain;f=libraries%2Fliblmdb%2Fmdb.c;h=2a3fb1a0dfb763614676d1b17c301f469bf644db;hb=110fba2ef1fcc5d08a78ed5ee72134828ce70b1a;hp=8b560ed32e238a71bff0529907232ecc630701a9;hpb=899b23f89dd9f0eeb795d0772c0a8cd046c84ec3;p=openldap diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index 8b560ed32e..2a3fb1a0df 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -5,7 +5,7 @@ * BerkeleyDB API, but much simplified. */ /* - * Copyright 2011-2014 Howard Chu, Symas Corp. + * Copyright 2011-2015 Howard Chu, Symas Corp. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -79,6 +79,15 @@ extern int cacheflush(char *addr, int nbytes, int cache); #define CACHEFLUSH(addr, bytes, cache) #endif +#if defined(__linux) && !defined(MDB_FDATASYNC_WORKS) +/** fdatasync is broken on ext3/ext4fs on older kernels, see + * description in #mdb_env_open2 comments. You can safely + * define MDB_FDATASYNC_WORKS if this code will only be run + * on kernels 3.6 and newer. + */ +#define BROKEN_FDATASYNC +#endif + #include #include #include @@ -89,6 +98,12 @@ extern int cacheflush(char *addr, int nbytes, int cache); #include #include +#if defined(__sun) || defined(ANDROID) +/* Most platforms have posix_memalign, older may only have memalign */ +#define HAVE_MEMALIGN 1 +#include +#endif + #if !(defined(BYTE_ORDER) || defined(__BYTE_ORDER)) #include #include /* defines BYTE_ORDER on HPUX and Solaris */ @@ -106,7 +121,14 @@ extern int cacheflush(char *addr, int nbytes, int cache); #ifdef MDB_USE_POSIX_SEM # define MDB_USE_HASH 1 #include +#else +#define MDB_USE_POSIX_MUTEX 1 +#endif #endif + +#if defined(_WIN32) + defined(MDB_USE_POSIX_SEM) \ + + defined(MDB_USE_POSIX_MUTEX) != 1 +# error "Ambiguous shared-lock implementation" #endif #ifdef USE_VALGRIND @@ -170,6 +192,12 @@ extern int cacheflush(char *addr, int nbytes, int cache); #define ESECT #endif +#ifdef _MSC_VER +#define CALL_CONV WINAPI +#else +#define CALL_CONV +#endif + /** @defgroup internal LMDB Internals * @{ */ @@ -196,6 +224,18 @@ extern int cacheflush(char *addr, int nbytes, int cache); # define mdb_func_ "" #endif +/* Internal error codes, not exposed outside liblmdb */ +#define MDB_NO_ROOT (MDB_LAST_ERRCODE + 10) +#ifdef _WIN32 +#define MDB_OWNERDEAD ((int) WAIT_ABANDONED) +#elif defined(MDB_USE_POSIX_MUTEX) && defined(EOWNERDEAD) +#define MDB_OWNERDEAD EOWNERDEAD /**< #LOCK_MUTEX0() result if dead owner */ +#endif + +#ifdef MDB_OWNERDEAD +#define MDB_ROBUST_SUPPORTED 1 +#endif + #ifdef _WIN32 #define MDB_USE_HASH 1 #define MDB_PIDLOCK 0 @@ -203,6 +243,7 @@ extern int cacheflush(char *addr, int nbytes, int cache); #define pthread_t HANDLE #define pthread_mutex_t HANDLE #define pthread_cond_t HANDLE +typedef HANDLE mdb_mutex_t, mdb_mutexref_t; #define pthread_key_t DWORD #define pthread_self() GetCurrentThreadId() #define pthread_key_create(x,y) \ @@ -216,10 +257,9 @@ extern int cacheflush(char *addr, int nbytes, int cache); #define pthread_cond_wait(cond,mutex) do{SignalObjectAndWait(*mutex, *cond, INFINITE, FALSE); WaitForSingleObject(*mutex, INFINITE);}while(0) #define THREAD_CREATE(thr,start,arg) thr=CreateThread(NULL,0,start,arg,0,NULL) #define THREAD_FINISH(thr) WaitForSingleObject(thr, INFINITE) -#define LOCK_MUTEX_R(env) pthread_mutex_lock(&(env)->me_rmutex) -#define UNLOCK_MUTEX_R(env) pthread_mutex_unlock(&(env)->me_rmutex) -#define LOCK_MUTEX_W(env) pthread_mutex_lock(&(env)->me_wmutex) -#define UNLOCK_MUTEX_W(env) pthread_mutex_unlock(&(env)->me_wmutex) +#define LOCK_MUTEX0(mutex) WaitForSingleObject(mutex, INFINITE) +#define UNLOCK_MUTEX(mutex) ReleaseMutex(mutex) +#define mdb_mutex_consistent(mutex) 0 #define getpid() GetCurrentProcessId() #define MDB_FDATASYNC(fd) (!FlushFileBuffers(fd)) #define MDB_MSYNC(addr,len,flags) (!FlushViewOfFile(addr,len)) @@ -244,10 +284,9 @@ extern int cacheflush(char *addr, int nbytes, int cache); #ifdef MDB_USE_POSIX_SEM -#define LOCK_MUTEX_R(env) mdb_sem_wait((env)->me_rmutex) -#define UNLOCK_MUTEX_R(env) sem_post((env)->me_rmutex) -#define LOCK_MUTEX_W(env) mdb_sem_wait((env)->me_wmutex) -#define UNLOCK_MUTEX_W(env) sem_post((env)->me_wmutex) +typedef sem_t *mdb_mutex_t, *mdb_mutexref_t; +#define LOCK_MUTEX0(mutex) mdb_sem_wait(mutex) +#define UNLOCK_MUTEX(mutex) sem_post(mutex) static int mdb_sem_wait(sem_t *sem) @@ -257,22 +296,26 @@ mdb_sem_wait(sem_t *sem) return rc; } -#else - /** Lock the reader mutex. +#else /* MDB_USE_POSIX_MUTEX: */ + /** Shared mutex/semaphore as it is stored (mdb_mutex_t), and as + * local variables keep it (mdb_mutexref_t). + * + * When #mdb_mutexref_t is a pointer declaration and #mdb_mutex_t is + * not, then it is array[size 1] so it can be assigned to a pointer. + * @{ */ -#define LOCK_MUTEX_R(env) pthread_mutex_lock(&(env)->me_txns->mti_mutex) - /** Unlock the reader mutex. +typedef pthread_mutex_t mdb_mutex_t[1], *mdb_mutexref_t; + /* @} */ + /** Lock the reader or writer mutex. + * Returns 0 or a code to give #mdb_mutex_failed(), as in #LOCK_MUTEX(). */ -#define UNLOCK_MUTEX_R(env) pthread_mutex_unlock(&(env)->me_txns->mti_mutex) - - /** Lock the writer mutex. - * Only a single write transaction is allowed at a time. Other writers - * will block waiting for this mutex. +#define LOCK_MUTEX0(mutex) pthread_mutex_lock(mutex) + /** Unlock the reader or writer mutex. */ -#define LOCK_MUTEX_W(env) pthread_mutex_lock(&(env)->me_txns->mti_wmutex) - /** Unlock the writer mutex. +#define UNLOCK_MUTEX(mutex) pthread_mutex_unlock(mutex) + /** Mark mutex-protected data as repaired, after death of previous owner. */ -#define UNLOCK_MUTEX_W(env) pthread_mutex_unlock(&(env)->me_txns->mti_wmutex) +#define mdb_mutex_consistent(mutex) pthread_mutex_consistent(mutex) #endif /* MDB_USE_POSIX_SEM */ /** Get the error code for the last failed system function. @@ -306,6 +349,19 @@ mdb_sem_wait(sem_t *sem) /** @} */ +#ifdef MDB_ROBUST_SUPPORTED + /** Lock mutex, handle any error, set rc = result. + * Return 0 on success, nonzero (not rc) on error. + */ +#define LOCK_MUTEX(rc, env, mutex) \ + (((rc) = LOCK_MUTEX0(mutex)) && \ + ((rc) = mdb_mutex_failed(env, mutex, rc))) +static int mdb_mutex_failed(MDB_env *env, mdb_mutexref_t mutex, int rc); +#else +#define LOCK_MUTEX(rc, env, mutex) ((rc) = LOCK_MUTEX0(mutex)) +#define mdb_mutex_failed(env, mutex, rc) (rc) +#endif + #ifndef _WIN32 /** A flag for opening a file and requesting synchronous data writes. * This is only used when writing a meta page. It's not strictly needed; @@ -431,12 +487,17 @@ static txnid_t mdb_debug_start; /** The version number for a database's lockfile format. */ #define MDB_LOCK_VERSION 1 - /** @brief The max size of a key we can write, or 0 for dynamic max. + /** @brief The max size of a key we can write, or 0 for computed max. + * + * This macro should normally be left alone or set to 0. + * Note that a database with big keys or dupsort data cannot be + * reliably modified by a liblmdb which uses a smaller max. + * The default is 511 for backwards compat, or 0 when #MDB_DEVEL. * - * Define this as 0 to compute the max from the page size. 511 - * is default for backwards compat: liblmdb <= 0.9.10 can break - * when modifying a DB with keys/dupsort data bigger than its max. - * #MDB_DEVEL sets the default to 0. + * Other values are allowed, for backwards compat. However: + * A value bigger than the computed max can break if you do not + * know what you are doing, and liblmdb <= 0.9.10 can break when + * modifying a DB with keys/dupsort data bigger than its max. * * Data items in an #MDB_DUPSORT database are also limited to * this size, since they're actually keys of a sub-DB. Keys and @@ -573,11 +634,11 @@ typedef struct MDB_rxbody { * started from so we can avoid overwriting any data used in that * particular version. */ - txnid_t mrb_txnid; + volatile txnid_t mrb_txnid; /** The process ID of the process owning this reader txn. */ - MDB_PID_T mrb_pid; + volatile MDB_PID_T mrb_pid; /** The thread ID of the thread owning this txn. */ - MDB_THR_T mrb_tid; + volatile MDB_THR_T mrb_tid; } MDB_rxbody; /** The actual reader record, with cacheline padding. */ @@ -617,20 +678,20 @@ typedef struct MDB_txbody { char mtb_rmname[MNAME_LEN]; #else /** Mutex protecting access to this table. - * This is the reader lock that #LOCK_MUTEX_R acquires. + * This is the reader table lock used with LOCK_MUTEX(). */ - pthread_mutex_t mtb_mutex; + mdb_mutex_t mtb_rmutex; #endif /** The ID of the last transaction committed to the database. * This is recorded here only for convenience; the value can always * be determined by reading the main database meta pages. */ - txnid_t mtb_txnid; + volatile txnid_t mtb_txnid; /** The number of slots that have been used in the reader table. * This always records the maximum count, it is not decremented * when readers release their slots. */ - unsigned mtb_numreaders; + volatile unsigned mtb_numreaders; } MDB_txbody; /** The actual reader table definition. */ @@ -639,7 +700,7 @@ typedef struct MDB_txninfo { MDB_txbody mtb; #define mti_magic mt1.mtb.mtb_magic #define mti_format mt1.mtb.mtb_format -#define mti_mutex mt1.mtb.mtb_mutex +#define mti_rmutex mt1.mtb.mtb_rmutex #define mti_rmname mt1.mtb.mtb_rmname #define mti_txnid mt1.mtb.mtb_txnid #define mti_numreaders mt1.mtb.mtb_numreaders @@ -650,7 +711,7 @@ typedef struct MDB_txninfo { char mt2_wmname[MNAME_LEN]; #define mti_wmname mt2.mt2_wmname #else - pthread_mutex_t mt2_wmutex; + mdb_mutex_t mt2_wmutex; #define mti_wmutex mt2.mt2_wmutex #endif char pad[(MNAME_LEN+CACHELINE-1) & ~(CACHELINE-1)]; @@ -744,7 +805,7 @@ typedef struct MDB_page { /** The number of overflow pages needed to store the given size. */ #define OVPAGES(size, psize) ((PAGEHDRSZ-1 + (size)) / (psize) + 1) - /** Link in #MDB_txn.%mt_loose_pages list */ + /** Link in #MDB_txn.%mt_loose_pgs list */ #define NEXT_LOOSE_PAGE(p) (*(MDB_page **)((p) + 2)) /** Header for a single key/data pair within a page. @@ -891,7 +952,7 @@ typedef struct MDB_meta { /** Stamp identifying this as an LMDB file. It must be set * to #MDB_MAGIC. */ uint32_t mm_magic; - /** Version number of this lock file. Must be set to #MDB_DATA_VERSION. */ + /** Version number of this file. Must be set to #MDB_DATA_VERSION. */ uint32_t mm_version; void *mm_address; /**< address for fixed mapping */ size_t mm_mapsize; /**< size of mmap region */ @@ -901,7 +962,7 @@ typedef struct MDB_meta { /** Any persistent environment flags. @ref mdb_env */ #define mm_flags mm_dbs[0].md_flags pgno_t mm_last_pg; /**< last used page in file */ - txnid_t mm_txnid; /**< txnid that committed this page */ + volatile txnid_t mm_txnid; /**< txnid that committed this page */ } MDB_meta; /** Buffer for a stack-allocated meta page. @@ -949,6 +1010,8 @@ struct MDB_txn { * in this transaction, linked through #NEXT_LOOSE_PAGE(page). */ MDB_page *mt_loose_pgs; + /* #Number of loose pages (#mt_loose_pgs) */ + int mt_loose_count; /** The sorted list of dirty pages we temporarily wrote to disk * because the dirty list was full. page numbers in here are * shifted left by 1, deleted slots have the LSB set. @@ -974,6 +1037,7 @@ struct MDB_txn { #define DB_STALE 0x02 /**< Named-DB record is older than txnID */ #define DB_NEW 0x04 /**< Named-DB handle opened in this txn */ #define DB_VALID 0x08 /**< DB handle is valid, see also #MDB_VALID */ +#define DB_USRVALID 0x10 /**< As #DB_VALID, but not set for #FREE_DBI */ /** @} */ /** In write txns, array of cursors for each DB */ MDB_cursor **mt_cursors; @@ -988,7 +1052,11 @@ struct MDB_txn { * @ingroup internal * @{ */ -#define MDB_TXN_RDONLY 0x01 /**< read-only transaction */ + /** #mdb_txn_begin() flags */ +#define MDB_TXN_BEGIN_FLAGS MDB_RDONLY +#define MDB_TXN_RDONLY MDB_RDONLY /**< read-only transaction */ + /* internal txn flags */ +#define MDB_TXN_WRITEMAP MDB_WRITEMAP /**< copy of #MDB_env flag in writers */ #define MDB_TXN_ERROR 0x02 /**< txn is unusable after an error */ #define MDB_TXN_DIRTY 0x04 /**< must write, even if dirty list is empty */ #define MDB_TXN_SPILLS 0x08 /**< txn or a parent has spilled pages */ @@ -1087,11 +1155,14 @@ struct MDB_env { #define MDB_ENV_ACTIVE 0x20000000U /** me_txkey is set */ #define MDB_ENV_TXKEY 0x10000000U + /** fdatasync is unreliable */ +#define MDB_FSYNCONLY 0x08000000U uint32_t me_flags; /**< @ref mdb_env */ unsigned int me_psize; /**< DB page size, inited from me_os_psize */ unsigned int me_os_psize; /**< OS page size, from #GET_PAGESIZE */ unsigned int me_maxreaders; /**< size of the reader table */ - unsigned int me_numreaders; /**< max numreaders set by this env */ + /** Max #MDB_txninfo.%mti_numreaders of interest to #mdb_env_close() */ + volatile int me_close_readers; MDB_dbi me_numdbs; /**< number of DBs opened */ MDB_dbi me_maxdbs; /**< size of the DB table */ MDB_PID_T me_pid; /**< process ID of this env */ @@ -1101,6 +1172,7 @@ struct MDB_env { MDB_meta *me_metas[2]; /**< pointers to the two meta pages */ void *me_pbuf; /**< scratch area for DUPSORT put() */ MDB_txn *me_txn; /**< current write transaction */ + MDB_txn *me_txn0; /**< prealloc'd write transaction */ size_t me_mapsize; /**< size of the data memory map */ off_t me_size; /**< current file size */ pgno_t me_maxpg; /**< me_mapsize / me_psize */ @@ -1108,6 +1180,7 @@ struct MDB_env { uint16_t *me_dbflags; /**< array of flags from MDB_db.md_flags */ unsigned int *me_dbiseqs; /**< array of dbi sequence numbers */ pthread_key_t me_txkey; /**< thread-key for readers */ + txnid_t me_pgoldest; /**< ID of oldest reader last time we looked */ MDB_pgstate me_pgstate; /**< state of old pages from freeDB */ # define me_pglast me_pgstate.mf_pglast # define me_pghead me_pgstate.mf_pghead @@ -1126,11 +1199,13 @@ struct MDB_env { int me_live_reader; /**< have liveness lock in reader table */ #ifdef _WIN32 int me_pidquery; /**< Used in OpenProcess */ - HANDLE me_rmutex; /* Windows mutexes don't reside in shared mem */ - HANDLE me_wmutex; -#elif defined(MDB_USE_POSIX_SEM) - sem_t *me_rmutex; /* Shared mutexes are not supported */ - sem_t *me_wmutex; +#endif +#ifdef MDB_USE_POSIX_MUTEX /* Posix mutexes reside in shared mem */ +# define me_rmutex me_txns->mti_rmutex /**< Shared reader lock */ +# define me_wmutex me_txns->mti_wmutex /**< Shared writer lock */ +#else + mdb_mutex_t me_rmutex; + mdb_mutex_t me_wmutex; #endif void *me_userctx; /**< User-settable context */ MDB_assert_func *me_assert_func; /**< Callback for assertion failures */ @@ -1153,8 +1228,8 @@ typedef struct MDB_ntxn { #define MAX_WRITE (0x80000000U >> (sizeof(ssize_t) == 4)) /** Check \b txn and \b dbi arguments to a function */ -#define TXN_DBI_EXIST(txn, dbi) \ - ((txn) && (dbi) < (txn)->mt_numdbs && ((txn)->mt_dbflags[dbi] & DB_VALID)) +#define TXN_DBI_EXIST(txn, dbi, validity) \ + ((txn) && (dbi)<(txn)->mt_numdbs && ((txn)->mt_dbflags[dbi] & (validity))) /** Check for misused \b dbi handles */ #define TXN_DBI_CHANGED(txn, dbi) \ @@ -1182,7 +1257,7 @@ static int mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, static int mdb_env_read_header(MDB_env *env, MDB_meta *meta); static int mdb_env_pick_meta(const MDB_env *env); static int mdb_env_write_meta(MDB_txn *txn); -#if !(defined(_WIN32) || defined(MDB_USE_POSIX_SEM)) /* Drop unused excl arg */ +#ifdef MDB_USE_POSIX_MUTEX /* Drop unused excl arg */ # define mdb_env_close0(env, excl) mdb_env_close1(env) #endif static void mdb_env_close0(MDB_env *env, int excl); @@ -1216,14 +1291,23 @@ static int mdb_cursor_last(MDB_cursor *mc, MDB_val *key, MDB_val *data); static void mdb_cursor_init(MDB_cursor *mc, MDB_txn *txn, MDB_dbi dbi, MDB_xcursor *mx); static void mdb_xcursor_init0(MDB_cursor *mc); static void mdb_xcursor_init1(MDB_cursor *mc, MDB_node *node); +static void mdb_xcursor_init2(MDB_cursor *mc, MDB_xcursor *src_mx, int force); static int mdb_drop0(MDB_cursor *mc, int subs); static void mdb_default_cmp(MDB_txn *txn, MDB_dbi dbi); +static int mdb_reader_check0(MDB_env *env, int rlocked, int *dead); /** @cond */ static MDB_cmp_func mdb_cmp_memn, mdb_cmp_memnr, mdb_cmp_int, mdb_cmp_cint, mdb_cmp_long; /** @endcond */ +/** Compare two items pointing at size_t's of unknown alignment. */ +#ifdef MISALIGNED_OK +# define mdb_cmp_clong mdb_cmp_long +#else +# define mdb_cmp_clong mdb_cmp_cint +#endif + #ifdef _WIN32 static SECURITY_DESCRIPTOR mdb_null_sd; static SECURITY_ATTRIBUTES mdb_all_sa; @@ -1246,7 +1330,7 @@ static char *const mdb_errstr[] = { "MDB_NOTFOUND: No matching key/data pair found", "MDB_PAGE_NOTFOUND: Requested page not found", "MDB_CORRUPTED: Located page was wrong type", - "MDB_PANIC: Update of meta page failed", + "MDB_PANIC: Update of meta page failed or environment had fatal error", "MDB_VERSION_MISMATCH: Database environment version mismatch", "MDB_INVALID: File is not an LMDB file", "MDB_MAP_FULL: Environment mapsize limit reached", @@ -1305,7 +1389,7 @@ mdb_strerror(int err) buf[0] = 0; FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, - NULL, err, 0, ptr, sizeof(buf), pad); + NULL, err, 0, ptr, sizeof(buf), (va_list *)pad); return ptr; #else return strerror(err); @@ -1537,7 +1621,12 @@ mdb_cmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b) int mdb_dcmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b) { - return txn->mt_dbxs[dbi].md_dcmp(a, b); + MDB_cmp_func *dcmp = txn->mt_dbxs[dbi].md_dcmp; +#if UINT_MAX < SIZE_MAX + if (dcmp == mdb_cmp_int && a->mv_size == sizeof(size_t)) + dcmp = mdb_cmp_clong; +#endif + return dcmp(a, b); } /** Allocate memory for a page. @@ -1631,10 +1720,11 @@ mdb_page_loose(MDB_cursor *mc, MDB_page *mp) { int loose = 0; pgno_t pgno = mp->mp_pgno; + MDB_txn *txn = mc->mc_txn; if ((mp->mp_flags & P_DIRTY) && mc->mc_dbi != FREE_DBI) { - if (mc->mc_txn->mt_parent) { - MDB_ID2 *dl = mc->mc_txn->mt_u.dirty_list; + if (txn->mt_parent) { + MDB_ID2 *dl = txn->mt_u.dirty_list; /* If txn has a parent, make sure the page is in our * dirty list. */ @@ -1643,7 +1733,7 @@ mdb_page_loose(MDB_cursor *mc, MDB_page *mp) if (x <= dl[0].mid && dl[x].mid == pgno) { if (mp != dl[x].mptr) { /* bad cursor? */ mc->mc_flags &= ~(C_INITIALIZED|C_EOF); - mc->mc_txn->mt_flags |= MDB_TXN_ERROR; + txn->mt_flags |= MDB_TXN_ERROR; return MDB_CORRUPTED; } /* ok, it's ours */ @@ -1658,11 +1748,12 @@ mdb_page_loose(MDB_cursor *mc, MDB_page *mp) if (loose) { DPRINTF(("loosen db %d page %"Z"u", DDBI(mc), mp->mp_pgno)); - NEXT_LOOSE_PAGE(mp) = mc->mc_txn->mt_loose_pgs; - mc->mc_txn->mt_loose_pgs = mp; + NEXT_LOOSE_PAGE(mp) = txn->mt_loose_pgs; + txn->mt_loose_pgs = mp; + txn->mt_loose_count++; mp->mp_flags |= P_LOOSE; } else { - int rc = mdb_midl_append(&mc->mc_txn->mt_free_pgs, pgno); + int rc = mdb_midl_append(&txn->mt_free_pgs, pgno); if (rc) return rc; } @@ -1894,7 +1985,7 @@ mdb_page_dirty(MDB_txn *txn, MDB_page *mp) MDB_ID2 mid; int rc, (*insert)(MDB_ID2L, MDB_ID2 *); - if (txn->mt_env->me_flags & MDB_WRITEMAP) { + if (txn->mt_flags & MDB_TXN_WRITEMAP) { insert = mdb_mid2l_append; } else { insert = mdb_mid2l_insert; @@ -1935,20 +2026,22 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) #else enum { Paranoid = 0, Max_retries = INT_MAX /*infinite*/ }; #endif - int rc, retry = num * 20; + int rc, retry = num * 60; MDB_txn *txn = mc->mc_txn; MDB_env *env = txn->mt_env; pgno_t pgno, *mop = env->me_pghead; - unsigned i, j, k, mop_len = mop ? mop[0] : 0, n2 = num-1; + unsigned i, j, mop_len = mop ? mop[0] : 0, n2 = num-1; MDB_page *np; txnid_t oldest = 0, last; MDB_cursor_op op; MDB_cursor m2; + int found_old = 0; /* If there are any loose pages, just use them */ if (num == 1 && txn->mt_loose_pgs) { np = txn->mt_loose_pgs; txn->mt_loose_pgs = NEXT_LOOSE_PAGE(np); + txn->mt_loose_count--; DPRINTF(("db %d use loose page %"Z"u", DDBI(mc), np->mp_pgno)); *mp = np; @@ -1966,7 +2059,7 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) for (op = MDB_FIRST;; op = MDB_NEXT) { MDB_val key, data; MDB_node *leaf; - pgno_t *idl, old_id, new_id; + pgno_t *idl; /* Seek a big enough contiguous page range. Prefer * pages at the tail, just truncating the list. @@ -1984,8 +2077,8 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) if (op == MDB_FIRST) { /* 1st iteration */ /* Prepare to fetch more and coalesce */ - oldest = mdb_find_oldest(txn); last = env->me_pglast; + oldest = env->me_pgoldest; mdb_cursor_init(&m2, txn, FREE_DBI, NULL); if (last) { op = MDB_SET_RANGE; @@ -2000,8 +2093,15 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) last++; /* Do not fetch more if the record will be too recent */ - if (oldest <= last) - break; + if (oldest <= last) { + if (!found_old) { + oldest = mdb_find_oldest(txn); + env->me_pgoldest = oldest; + found_old = 1; + } + if (oldest <= last) + break; + } rc = mdb_cursor_get(&m2, &key, NULL, op); if (rc) { if (rc == MDB_NOTFOUND) @@ -2009,8 +2109,15 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) goto fail; } last = *(txnid_t*)key.mv_data; - if (oldest <= last) - break; + if (oldest <= last) { + if (!found_old) { + oldest = mdb_find_oldest(txn); + env->me_pgoldest = oldest; + found_old = 1; + } + if (oldest <= last) + break; + } np = m2.mc_pg[m2.mc_top]; leaf = NODEPTR(np, m2.mc_ki[m2.mc_top]); if ((rc = mdb_node_read(txn, leaf, &data)) != MDB_SUCCESS) @@ -2032,21 +2139,12 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) #if (MDB_DEBUG) > 1 DPRINTF(("IDL read txn %"Z"u root %"Z"u num %u", last, txn->mt_dbs[FREE_DBI].md_root, i)); - for (k = i; k; k--) - DPRINTF(("IDL %"Z"u", idl[k])); + for (j = i; j; j--) + DPRINTF(("IDL %"Z"u", idl[j])); #endif /* Merge in descending sorted order */ - j = mop_len; - k = mop_len += i; - mop[0] = (pgno_t)-1; - old_id = mop[j]; - while (i) { - new_id = idl[i--]; - for (; old_id < new_id; old_id = mop[--j]) - mop[k--] = old_id; - mop[k--] = new_id; - } - mop[0] = mop_len; + mdb_midl_xmerge(mop, idl); + mop_len = mop[0]; } /* Use new pages from the map when nothing suitable in the freeDB */ @@ -2282,6 +2380,8 @@ int mdb_env_sync(MDB_env *env, int force) { int rc = 0; + if (env->me_flags & MDB_RDONLY) + return EACCES; if (force || !F_ISSET(env->me_flags, MDB_NOSYNC)) { if (env->me_flags & MDB_WRITEMAP) { int flags = ((env->me_flags & MDB_MAPASYNC) && !force) @@ -2293,6 +2393,12 @@ mdb_env_sync(MDB_env *env, int force) rc = ErrCode(); #endif } else { +#ifdef BROKEN_FDATASYNC + if (env->me_flags & MDB_FSYNCONLY) { + if (fsync(env->me_fd)) + rc = ErrCode(); + } else +#endif if (MDB_FDATASYNC(env->me_fd)) rc = ErrCode(); } @@ -2448,15 +2554,11 @@ mdb_txn_renew0(MDB_txn *txn) MDB_env *env = txn->mt_env; MDB_txninfo *ti = env->me_txns; MDB_meta *meta; - unsigned int i, nr; + unsigned int i, nr, flags = txn->mt_flags; uint16_t x; int rc, new_notls = 0; - /* Setup db info */ - txn->mt_numdbs = env->me_numdbs; - txn->mt_dbxs = env->me_dbxs; /* mostly static anyway */ - - if (txn->mt_flags & MDB_TXN_RDONLY) { + if ((flags &= MDB_TXN_RDONLY) != 0) { if (!ti) { meta = env->me_metas[ mdb_env_pick_meta(env) ]; txn->mt_txnid = meta->mm_txnid; @@ -2470,6 +2572,7 @@ mdb_txn_renew0(MDB_txn *txn) } else { MDB_PID_T pid = env->me_pid; MDB_THR_T tid = pthread_self(); + mdb_mutexref_t rmutex = env->me_rmutex; if (!env->me_live_reader) { rc = mdb_reader_pid(env, Pidset, pid); @@ -2478,38 +2581,51 @@ mdb_txn_renew0(MDB_txn *txn) env->me_live_reader = 1; } - LOCK_MUTEX_R(env); + if (LOCK_MUTEX(rc, env, rmutex)) + return rc; nr = ti->mti_numreaders; for (i=0; imti_readers[i].mr_pid == 0) break; if (i == env->me_maxreaders) { - UNLOCK_MUTEX_R(env); + UNLOCK_MUTEX(rmutex); return MDB_READERS_FULL; } - ti->mti_readers[i].mr_pid = pid; - ti->mti_readers[i].mr_tid = tid; + r = &ti->mti_readers[i]; + /* Claim the reader slot, carefully since other code + * uses the reader table un-mutexed: First reset the + * slot, next publish it in mti_numreaders. After + * that, it is safe for mdb_env_close() to touch it. + * When it will be closed, we can finally claim it. + */ + r->mr_pid = 0; + r->mr_txnid = (txnid_t)-1; + r->mr_tid = tid; if (i == nr) ti->mti_numreaders = ++nr; - /* Save numreaders for un-mutexed mdb_env_close() */ - env->me_numreaders = nr; - UNLOCK_MUTEX_R(env); + env->me_close_readers = nr; + r->mr_pid = pid; + UNLOCK_MUTEX(rmutex); - r = &ti->mti_readers[i]; new_notls = (env->me_flags & MDB_NOTLS); if (!new_notls && (rc=pthread_setspecific(env->me_txkey, r))) { r->mr_pid = 0; return rc; } } - txn->mt_txnid = r->mr_txnid = ti->mti_txnid; + do /* LY: Retry on a race, ITS#7970. */ + r->mr_txnid = ti->mti_txnid; + while(r->mr_txnid != ti->mti_txnid); + txn->mt_txnid = r->mr_txnid; txn->mt_u.reader = r; meta = env->me_metas[txn->mt_txnid & 1]; } + txn->mt_dbxs = env->me_dbxs; /* mostly static anyway */ } else { + /* Not yet touching txn == env->me_txn0, it may be active */ if (ti) { - LOCK_MUTEX_W(env); - + if (LOCK_MUTEX(rc, env, env->me_wmutex)) + return rc; txn->mt_txnid = ti->mti_txnid; meta = env->me_metas[txn->mt_txnid & 1]; } else { @@ -2521,6 +2637,9 @@ mdb_txn_renew0(MDB_txn *txn) if (txn->mt_txnid == mdb_debug_start) mdb_debug = 1; #endif + txn->mt_child = NULL; + txn->mt_loose_pgs = NULL; + txn->mt_loose_count = 0; txn->mt_dirty_room = MDB_IDL_UM_MAX; txn->mt_u.dirty_list = env->me_dirty_list; txn->mt_u.dirty_list[0].mid = 0; @@ -2537,12 +2656,17 @@ mdb_txn_renew0(MDB_txn *txn) /* Moved to here to avoid a data race in read TXNs */ txn->mt_next_pgno = meta->mm_last_pg+1; + txn->mt_flags = flags; + + /* Setup db info */ + txn->mt_numdbs = env->me_numdbs; for (i=2; imt_numdbs; i++) { x = env->me_dbflags[i]; txn->mt_dbs[i].md_flags = x & PERSISTENT_FLAGS; - txn->mt_dbflags[i] = (x & MDB_VALID) ? DB_VALID|DB_STALE : 0; + txn->mt_dbflags[i] = (x & MDB_VALID) ? DB_VALID|DB_USRVALID|DB_STALE : 0; } - txn->mt_dbflags[0] = txn->mt_dbflags[1] = DB_VALID; + txn->mt_dbflags[MAIN_DBI] = DB_VALID|DB_USRVALID; + txn->mt_dbflags[FREE_DBI] = DB_VALID; if (env->me_maxpg < txn->mt_next_pgno) { mdb_txn_reset0(txn, "renew0-mapfail"); @@ -2583,56 +2707,52 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret) { MDB_txn *txn; MDB_ntxn *ntxn; - int rc, size, tsize = sizeof(MDB_txn); + int rc, size, tsize; + + flags &= MDB_TXN_BEGIN_FLAGS; + flags |= env->me_flags & MDB_WRITEMAP; if (env->me_flags & MDB_FATAL_ERROR) { DPUTS("environment had fatal error, must shutdown!"); return MDB_PANIC; } - if ((env->me_flags & MDB_RDONLY) && !(flags & MDB_RDONLY)) + if (env->me_flags & MDB_RDONLY & ~flags) /* write txn in RDONLY env */ return EACCES; + if (parent) { /* Nested transactions: Max 1 child, write txns only, no writemap */ + flags |= parent->mt_flags; if (parent->mt_child || - (flags & MDB_RDONLY) || - (parent->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_ERROR)) || - (env->me_flags & MDB_WRITEMAP)) + (flags & (MDB_RDONLY|MDB_WRITEMAP|MDB_TXN_ERROR))) { return (parent->mt_flags & MDB_TXN_RDONLY) ? EINVAL : MDB_BAD_TXN; } - tsize = sizeof(MDB_ntxn); - } - size = tsize + env->me_maxdbs * (sizeof(MDB_db)+1); - if (!(flags & MDB_RDONLY)) { - size += env->me_maxdbs * sizeof(MDB_cursor *); - /* child txns use parent's dbiseqs */ - if (!parent) - size += env->me_maxdbs * sizeof(unsigned int); + /* Child txns save MDB_pgstate and use own copy of cursors */ + size = env->me_maxdbs * (sizeof(MDB_db)+sizeof(MDB_cursor *)+1); + size += tsize = sizeof(MDB_ntxn); + } else if (flags & MDB_RDONLY) { + size = env->me_maxdbs * (sizeof(MDB_db)+1); + size += tsize = sizeof(MDB_txn); + } else { + /* Reuse preallocated write txn. However, do not touch it until + * mdb_txn_renew0() succeeds, since it currently may be active. + */ + txn = env->me_txn0; + goto renew; } - if ((txn = calloc(1, size)) == NULL) { DPRINTF(("calloc: %s", strerror(errno))); return ENOMEM; } txn->mt_dbs = (MDB_db *) ((char *)txn + tsize); - if (flags & MDB_RDONLY) { - txn->mt_flags |= MDB_TXN_RDONLY; - txn->mt_dbflags = (unsigned char *)(txn->mt_dbs + env->me_maxdbs); - txn->mt_dbiseqs = env->me_dbiseqs; - } else { - txn->mt_cursors = (MDB_cursor **)(txn->mt_dbs + env->me_maxdbs); - if (parent) { - txn->mt_dbiseqs = parent->mt_dbiseqs; - txn->mt_dbflags = (unsigned char *)(txn->mt_cursors + env->me_maxdbs); - } else { - txn->mt_dbiseqs = (unsigned int *)(txn->mt_cursors + env->me_maxdbs); - txn->mt_dbflags = (unsigned char *)(txn->mt_dbiseqs + env->me_maxdbs); - } - } + txn->mt_dbflags = (unsigned char *)txn + size - env->me_maxdbs; + txn->mt_flags = flags; txn->mt_env = env; if (parent) { unsigned int i; + txn->mt_cursors = (MDB_cursor **)(txn->mt_dbs + env->me_maxdbs); + txn->mt_dbiseqs = parent->mt_dbiseqs; txn->mt_u.dirty_list = malloc(sizeof(MDB_ID2)*MDB_IDL_UM_SIZE); if (!txn->mt_u.dirty_list || !(txn->mt_free_pgs = mdb_midl_alloc(MDB_IDL_UM_MAX))) @@ -2649,7 +2769,6 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret) parent->mt_child = txn; txn->mt_parent = parent; txn->mt_numdbs = parent->mt_numdbs; - txn->mt_flags = parent->mt_flags; txn->mt_dbxs = parent->mt_dbxs; memcpy(txn->mt_dbs, parent->mt_dbs, txn->mt_numdbs * sizeof(MDB_db)); /* Copy parent's mt_dbflags, but clear DB_NEW */ @@ -2670,15 +2789,19 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret) rc = mdb_cursor_shadow(parent, txn); if (rc) mdb_txn_reset0(txn, "beginchild-fail"); - } else { + } else { /* MDB_RDONLY */ + txn->mt_dbiseqs = env->me_dbiseqs; +renew: rc = mdb_txn_renew0(txn); } - if (rc) - free(txn); - else { + if (rc) { + if (txn != env->me_txn0) + free(txn); + } else { + txn->mt_flags |= flags; /* could not change txn=me_txn0 earlier */ *ret = txn; DPRINTF(("begin txn %"Z"u%c %p on mdbenv %p, root page %"Z"u", - txn->mt_txnid, (txn->mt_flags & MDB_TXN_RDONLY) ? 'r' : 'w', + txn->mt_txnid, (flags & MDB_RDONLY) ? 'r' : 'w', (void *) txn, (void *) env, txn->mt_dbs[MAIN_DBI].md_root)); } @@ -2692,6 +2815,13 @@ mdb_txn_env(MDB_txn *txn) return txn->mt_env; } +size_t +mdb_txn_id(MDB_txn *txn) +{ + if(!txn) return 0; + return txn->mt_txnid; +} + /** Export or close DBI handles opened in this txn. */ static void mdb_dbis_update(MDB_txn *txn, int keep) @@ -2747,31 +2877,33 @@ mdb_txn_reset0(MDB_txn *txn, const char *act) txn->mt_numdbs = 0; /* close nothing if called again */ txn->mt_dbxs = NULL; /* mark txn as reset */ } else { - mdb_cursors_close(txn, 0); + pgno_t *pghead = env->me_pghead; + mdb_cursors_close(txn, 0); if (!(env->me_flags & MDB_WRITEMAP)) { mdb_dlist_free(txn); } - mdb_midl_free(env->me_pghead); - if (txn->mt_parent) { + if (!txn->mt_parent) { + mdb_midl_shrink(&txn->mt_free_pgs); + env->me_free_pgs = txn->mt_free_pgs; + /* me_pgstate: */ + env->me_pghead = NULL; + env->me_pglast = 0; + + env->me_txn = NULL; + /* The writer mutex was locked in mdb_txn_begin. */ + if (env->me_txns) + UNLOCK_MUTEX(env->me_wmutex); + } else { txn->mt_parent->mt_child = NULL; env->me_pgstate = ((MDB_ntxn *)txn)->mnt_pgstate; mdb_midl_free(txn->mt_free_pgs); mdb_midl_free(txn->mt_spill_pgs); free(txn->mt_u.dirty_list); - return; } - if (mdb_midl_shrink(&txn->mt_free_pgs)) - env->me_free_pgs = txn->mt_free_pgs; - env->me_pghead = NULL; - env->me_pglast = 0; - - env->me_txn = NULL; - /* The writer mutex was locked in mdb_txn_begin. */ - if (env->me_txns) - UNLOCK_MUTEX_W(env); + mdb_midl_free(pghead); } } @@ -2802,7 +2934,8 @@ mdb_txn_abort(MDB_txn *txn) if ((txn->mt_flags & MDB_TXN_RDONLY) && txn->mt_u.reader) txn->mt_u.reader->mr_pid = 0; - free(txn); + if (txn != txn->mt_env->me_txn0) + free(txn); } /** Save the freelist as of this transaction to the freeDB. @@ -2831,30 +2964,17 @@ mdb_freelist_save(MDB_txn *txn) return rc; } - /* Dispose of loose pages. Usually they will have all - * been used up by the time we get here. - */ - if (txn->mt_loose_pgs) { + if (!env->me_pghead && txn->mt_loose_pgs) { + /* Put loose page numbers in mt_free_pgs, since + * we may be unable to return them to me_pghead. + */ MDB_page *mp = txn->mt_loose_pgs; - /* Just return them to freeDB */ - if (env->me_pghead) { - int i, j; - mop = env->me_pghead; - for (; mp; mp = NEXT_LOOSE_PAGE(mp)) { - pgno_t pg = mp->mp_pgno; - j = mop[0] + 1; - for (i = mop[0]; i && mop[i] < pg; i--) - mop[j--] = mop[i]; - mop[j] = pg; - mop[0] += 1; - } - } else { - /* Oh well, they were wasted. Put on freelist */ - for (; mp; mp = NEXT_LOOSE_PAGE(mp)) { - mdb_midl_append(&txn->mt_free_pgs, mp->mp_pgno); - } - } + if ((rc = mdb_midl_need(&txn->mt_free_pgs, txn->mt_loose_count)) != 0) + return rc; + for (; mp; mp = NEXT_LOOSE_PAGE(mp)) + mdb_midl_xappend(txn->mt_free_pgs, mp->mp_pgno); txn->mt_loose_pgs = NULL; + txn->mt_loose_count = 0; } /* MDB_RESERVE cancels meminit in ovpage malloc (when no WRITEMAP) */ @@ -2918,7 +3038,7 @@ mdb_freelist_save(MDB_txn *txn) } mop = env->me_pghead; - mop_len = mop ? mop[0] : 0; + mop_len = (mop ? mop[0] : 0) + txn->mt_loose_count; /* Reserve records for me_pghead[]. Split it if multi-page, * to avoid searching freeDB for a page range. Use keys in @@ -2958,6 +3078,28 @@ mdb_freelist_save(MDB_txn *txn) total_room += head_room; } + /* Return loose page numbers to me_pghead, though usually none are + * left at this point. The pages themselves remain in dirty_list. + */ + if (txn->mt_loose_pgs) { + MDB_page *mp = txn->mt_loose_pgs; + unsigned count = txn->mt_loose_count; + MDB_IDL loose; + /* Room for loose pages + temp IDL with same */ + if ((rc = mdb_midl_need(&env->me_pghead, 2*count+1)) != 0) + return rc; + mop = env->me_pghead; + loose = mop + MDB_IDL_ALLOCLEN(mop) - count; + for (count = 0; mp; mp = NEXT_LOOSE_PAGE(mp)) + loose[ ++count ] = mp->mp_pgno; + loose[0] = count; + mdb_midl_sort(loose); + mdb_midl_xmerge(mop, loose); + txn->mt_loose_pgs = NULL; + txn->mt_loose_count = 0; + mop_len = mop[0]; + } + /* Fill in the reserved me_pghead records */ rc = MDB_SUCCESS; if (mop_len) { @@ -3069,6 +3211,7 @@ mdb_page_flush(MDB_txn *txn, int keep) /* Write up to MDB_COMMIT_PAGES dirty pages at a time. */ if (pos!=next_pos || n==MDB_COMMIT_PAGES || wsize+size>MAX_WRITE) { if (n) { +retry_write: /* Write previous page(s) */ #ifdef MDB_USE_PWRITEV wres = pwritev(env->me_fd, iov, n, wpos); @@ -3076,8 +3219,11 @@ mdb_page_flush(MDB_txn *txn, int keep) if (n == 1) { wres = pwrite(env->me_fd, iov[0].iov_base, wsize, wpos); } else { +retry_seek: if (lseek(env->me_fd, wpos, SEEK_SET) == -1) { rc = ErrCode(); + if (rc == EINTR) + goto retry_seek; DPRINTF(("lseek: %s", strerror(rc))); return rc; } @@ -3087,6 +3233,8 @@ mdb_page_flush(MDB_txn *txn, int keep) if (wres != wsize) { if (wres < 0) { rc = ErrCode(); + if (rc == EINTR) + goto retry_write; DPRINTF(("Write error: %s", strerror(rc))); } else { rc = EIO; /* TODO: Use which error code? */ @@ -3141,12 +3289,11 @@ mdb_txn_commit(MDB_txn *txn) unsigned int i; MDB_env *env; - if (txn == NULL || txn->mt_env == NULL) + if (txn == NULL) return EINVAL; if (txn->mt_child) { rc = mdb_txn_commit(txn->mt_child); - txn->mt_child = NULL; if (rc) goto fail; } @@ -3272,6 +3419,7 @@ mdb_txn_commit(MDB_txn *txn) for (lp = &parent->mt_loose_pgs; *lp; lp = &NEXT_LOOSE_PAGE(lp)) ; *lp = txn->mt_loose_pgs; + parent->mt_loose_count += txn->mt_loose_count; parent->mt_child = NULL; mdb_midl_free(((MDB_ntxn *)txn)->mnt_pgstate.mf_pghead); @@ -3309,7 +3457,8 @@ mdb_txn_commit(MDB_txn *txn) goto fail; } data.mv_data = &txn->mt_dbs[i]; - rc = mdb_cursor_put(&mc, &txn->mt_dbxs[i].md_name, &data, 0); + rc = mdb_cursor_put(&mc, &txn->mt_dbxs[i].md_name, &data, + F_SUBDATA); if (rc) goto fail; } @@ -3322,8 +3471,8 @@ mdb_txn_commit(MDB_txn *txn) mdb_midl_free(env->me_pghead); env->me_pghead = NULL; - if (mdb_midl_shrink(&txn->mt_free_pgs)) - env->me_free_pgs = txn->mt_free_pgs; + mdb_midl_shrink(&txn->mt_free_pgs); + env->me_free_pgs = txn->mt_free_pgs; #if (MDB_DEBUG) > 2 mdb_audit(txn); @@ -3334,14 +3483,19 @@ mdb_txn_commit(MDB_txn *txn) (rc = mdb_env_write_meta(txn))) goto fail; + /* Free P_LOOSE pages left behind in dirty_list */ + if (!(env->me_flags & MDB_WRITEMAP)) + mdb_dlist_free(txn); + done: env->me_pglast = 0; env->me_txn = NULL; mdb_dbis_update(txn, 1); if (env->me_txns) - UNLOCK_MUTEX_W(env); - free(txn); + UNLOCK_MUTEX(env->me_wmutex); + if (txn != env->me_txn0) + free(txn); return MDB_SUCCESS; @@ -3414,6 +3568,7 @@ mdb_env_read_header(MDB_env *env, MDB_meta *meta) return 0; } +/** Fill in most of the zeroed #MDB_meta for an empty database environment */ static void ESECT mdb_env_init_meta0(MDB_env *env, MDB_meta *meta) { @@ -3430,7 +3585,7 @@ mdb_env_init_meta0(MDB_env *env, MDB_meta *meta) /** Write the environment parameters of a freshly created DB environment. * @param[in] env the environment handle - * @param[out] meta address of where to store the meta information + * @param[in] meta the #MDB_meta to write * @return 0 on success, non-zero on failure. */ static int ESECT @@ -3450,16 +3605,18 @@ mdb_env_init_meta(MDB_env *env, MDB_meta *meta) int len; #define DO_PWRITE(rc, fd, ptr, size, len, pos) do { \ len = pwrite(fd, ptr, size, pos); \ - rc = (len >= 0); } while(0) + if (len == -1 && ErrCode() == EINTR) continue; \ + rc = (len >= 0); break; } while(1) #endif DPUTS("writing new meta page"); psize = env->me_psize; - mdb_env_init_meta0(env, meta); - p = calloc(2, psize); + if (!p) + return ENOMEM; + p->mp_pgno = 0; p->mp_flags = P_META; *(MDB_meta *)METADATA(p) = *meta; @@ -3489,6 +3646,7 @@ mdb_env_write_meta(MDB_txn *txn) { MDB_env *env; MDB_meta meta, metab, *mp; + unsigned flags; size_t mapsize; off_t off; int rc, len, toggle; @@ -3505,19 +3663,25 @@ mdb_env_write_meta(MDB_txn *txn) toggle, txn->mt_dbs[MAIN_DBI].md_root)); env = txn->mt_env; + flags = env->me_flags; mp = env->me_metas[toggle]; mapsize = env->me_metas[toggle ^ 1]->mm_mapsize; /* Persist any increases of mapsize config */ if (mapsize < env->me_mapsize) mapsize = env->me_mapsize; - if (env->me_flags & MDB_WRITEMAP) { + if (flags & MDB_WRITEMAP) { mp->mm_mapsize = mapsize; mp->mm_dbs[0] = txn->mt_dbs[0]; mp->mm_dbs[1] = txn->mt_dbs[1]; mp->mm_last_pg = txn->mt_next_pgno - 1; +#if (__GNUC__ * 100 + __GNUC_MINOR__ >= 404) && /* TODO: portability */ \ + !(defined(__i386__) || defined(__x86_64__)) + /* LY: issue a memory barrier, if not x86. ITS#7969 */ + __sync_synchronize(); +#endif mp->mm_txnid = txn->mt_txnid; - if (!(env->me_flags & (MDB_NOMETASYNC|MDB_NOSYNC))) { + if (!(flags & (MDB_NOMETASYNC|MDB_NOSYNC))) { unsigned meta_size = env->me_psize; rc = (env->me_flags & MDB_MAPASYNC) ? MS_ASYNC : MS_SYNC; ptr = env->me_map; @@ -3553,8 +3717,7 @@ mdb_env_write_meta(MDB_txn *txn) off += PAGEHDRSZ; /* Write to the SYNC fd */ - mfd = env->me_flags & (MDB_NOSYNC|MDB_NOMETASYNC) ? - env->me_fd : env->me_mfd; + mfd = (flags & (MDB_NOSYNC|MDB_NOMETASYNC)) ? env->me_fd : env->me_mfd; #ifdef _WIN32 { memset(&ov, 0, sizeof(ov)); @@ -3563,10 +3726,15 @@ mdb_env_write_meta(MDB_txn *txn) rc = -1; } #else +retry_write: rc = pwrite(mfd, ptr, len, off); #endif if (rc != len) { rc = rc < 0 ? ErrCode() : EIO; +#ifndef _WIN32 + if (rc == EINTR) + goto retry_write; +#endif DPUTS("write failed, disk error?"); /* On a failure, the pagecache still contains the new data. * Write some old data back, to prevent it from being used. @@ -3586,11 +3754,9 @@ fail: env->me_flags |= MDB_FATAL_ERROR; return rc; } -done: /* MIPS has cache coherency issues, this is a no-op everywhere else */ - if (!(env->me_flags & MDB_WRITEMAP)) { - CACHEFLUSH(env->me_map + off, len, DCACHE); - } + CACHEFLUSH(env->me_map + off, len, DCACHE); +done: /* Memory ordering issues are irrelevant; since the entire writer * is wrapped by wmutex, all of these changes will become visible * after the wmutex is unlocked. Since the DB is multi-version, @@ -3730,16 +3896,16 @@ mdb_env_set_mapsize(MDB_env *env, size_t size) */ if (env->me_map) { int rc; + MDB_meta *meta; void *old; if (env->me_txn) return EINVAL; + meta = env->me_metas[mdb_env_pick_meta(env)]; if (!size) - size = env->me_metas[mdb_env_pick_meta(env)]->mm_mapsize; - else if (size < env->me_mapsize) { - /* If the configured size is smaller, make sure it's - * still big enough. Silently round up to minimum if not. - */ - size_t minsize = (env->me_metas[mdb_env_pick_meta(env)]->mm_last_pg + 1) * env->me_psize; + size = meta->mm_mapsize; + { + /* Silently round up to minimum if the size is too small */ + size_t minsize = (meta->mm_last_pg + 1) * env->me_psize; if (size < minsize) size = minsize; } @@ -3783,6 +3949,32 @@ mdb_env_get_maxreaders(MDB_env *env, unsigned int *readers) return MDB_SUCCESS; } +static int ESECT +mdb_fsize(HANDLE fd, size_t *size) +{ +#ifdef _WIN32 + LARGE_INTEGER fsize; + + if (!GetFileSizeEx(fd, &fsize)) + return ErrCode(); + + *size = fsize.QuadPart; +#else + struct stat st; + + if (fstat(fd, &st)) + return ErrCode(); + + *size = st.st_size; +#endif + return MDB_SUCCESS; +} + +#ifdef BROKEN_FDATASYNC +#include +#include +#endif + /** Further setup required for opening an LMDB environment */ static int ESECT @@ -3801,7 +3993,53 @@ mdb_env_open2(MDB_env *env) env->me_pidquery = PROCESS_QUERY_INFORMATION; #endif /* _WIN32 */ - memset(&meta, 0, sizeof(meta)); +#ifdef BROKEN_FDATASYNC + /* ext3/ext4 fdatasync is broken on some older Linux kernels. + * https://lkml.org/lkml/2012/9/3/83 + * Kernels after 3.6-rc6 are known good. + * https://lkml.org/lkml/2012/9/10/556 + * See if the DB is on ext3/ext4, then check for new enough kernel + * Kernels 2.6.32.60, 2.6.34.15, 3.2.30, and 3.5.4 are also known + * to be patched. + */ + { + struct statfs st; + fstatfs(env->me_fd, &st); + while (st.f_type == 0xEF53) { + struct utsname uts; + int i; + uname(&uts); + if (uts.release[0] < '3') { + if (!strncmp(uts.release, "2.6.32.", 7)) { + i = atoi(uts.release+7); + if (i >= 60) + break; /* 2.6.32.60 and newer is OK */ + } else if (!strncmp(uts.release, "2.6.34.", 7)) { + i = atoi(uts.release+7); + if (i >= 15) + break; /* 2.6.34.15 and newer is OK */ + } + } else if (uts.release[0] == '3') { + i = atoi(uts.release+2); + if (i > 5) + break; /* 3.6 and newer is OK */ + if (i == 5) { + i = atoi(uts.release+4); + if (i >= 4) + break; /* 3.5.4 and newer is OK */ + } else if (i == 2) { + i = atoi(uts.release+4); + if (i >= 30) + break; /* 3.2.30 and newer is OK */ + } + } else { /* 4.x and newer is OK */ + break; + } + env->me_flags |= MDB_FSYNCONLY; + break; + } + } +#endif if ((i = mdb_env_read_header(env, &meta)) != 0) { if (i != ENOENT) @@ -3811,24 +4049,40 @@ mdb_env_open2(MDB_env *env) env->me_psize = env->me_os_psize; if (env->me_psize > MAX_PAGESIZE) env->me_psize = MAX_PAGESIZE; + memset(&meta, 0, sizeof(meta)); + mdb_env_init_meta0(env, &meta); + meta.mm_mapsize = DEFAULT_MAPSIZE; } else { env->me_psize = meta.mm_psize; } /* Was a mapsize configured? */ if (!env->me_mapsize) { - /* If this is a new environment, take the default, - * else use the size recorded in the existing env. - */ - env->me_mapsize = newenv ? DEFAULT_MAPSIZE : meta.mm_mapsize; - } else if (env->me_mapsize < meta.mm_mapsize) { - /* If the configured size is smaller, make sure it's - * still big enough. Silently round up to minimum if not. + env->me_mapsize = meta.mm_mapsize; + } + { + /* Make sure mapsize >= committed data size. Even when using + * mm_mapsize, which could be broken in old files (ITS#7789). */ size_t minsize = (meta.mm_last_pg + 1) * meta.mm_psize; if (env->me_mapsize < minsize) env->me_mapsize = minsize; } + meta.mm_mapsize = env->me_mapsize; + + if (newenv && !(flags & MDB_FIXEDMAP)) { + /* mdb_env_map() may grow the datafile. Write the metapages + * first, so the file will be valid if initialization fails. + * Except with FIXEDMAP, since we do not yet know mm_address. + * We could fill in mm_address later, but then a different + * program might end up doing that - one with a memory layout + * and map address which does not suit the main program. + */ + rc = mdb_env_init_meta(env, &meta); + if (rc) + return rc; + newenv = 0; + } rc = mdb_env_map(env, (flags & MDB_FIXEDMAP) ? meta.mm_address : NULL); if (rc) @@ -3931,7 +4185,7 @@ PIMAGE_TLS_CALLBACK mdb_tls_cbp __attribute__((section (".CRT$XLB"))) = mdb_tls_ extern const PIMAGE_TLS_CALLBACK mdb_tls_cbp; const PIMAGE_TLS_CALLBACK mdb_tls_cbp = mdb_tls_callback; #pragma const_seg() -#else /* WIN32 */ +#else /* _WIN32 */ #pragma comment(linker, "/INCLUDE:__tls_used") #pragma comment(linker, "/INCLUDE:_mdb_tls_cbp") #pragma data_seg(".CRT$XLB") @@ -3981,7 +4235,7 @@ mdb_env_share_locks(MDB_env *env, int *excl) return rc; } -/** Try to get exlusive lock, otherwise shared. +/** Try to get exclusive lock, otherwise shared. * Maintain *excl = -1: no/unknown lock, 0: shared, 1: exclusive. */ static int ESECT @@ -4012,8 +4266,8 @@ mdb_env_excl_lock(MDB_env *env, int *excl) if (!rc) { *excl = 1; } else -# ifdef MDB_USE_POSIX_SEM - if (*excl < 0) /* always true when !MDB_USE_POSIX_SEM */ +# ifndef MDB_USE_POSIX_MUTEX + if (*excl < 0) /* always true when MDB_USE_POSIX_MUTEX */ # endif { lock_info.l_type = F_RDLCK; @@ -4122,7 +4376,6 @@ mdb_hash_enc(MDB_val *val, char *encbuf) * @param[in] env The LMDB environment. * @param[in] lpath The pathname of the file used for the lock region. * @param[in] mode The Unix permissions for the file, if we create it. - * @param[out] excl Resulting file lock type: -1 none, 0 shared, 1 exclusive * @param[in,out] excl In -1, out lock type: -1 none, 0 shared, 1 exclusive * @return 0 on success, non-zero on failure. */ @@ -4286,13 +4539,16 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl) env->me_wmutex = sem_open(env->me_txns->mti_wmname, O_CREAT|O_EXCL, mode, 1); if (env->me_wmutex == SEM_FAILED) goto fail_errno; -#else /* MDB_USE_POSIX_SEM */ +#else /* MDB_USE_POSIX_MUTEX: */ pthread_mutexattr_t mattr; if ((rc = pthread_mutexattr_init(&mattr)) || (rc = pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED)) - || (rc = pthread_mutex_init(&env->me_txns->mti_mutex, &mattr)) - || (rc = pthread_mutex_init(&env->me_txns->mti_wmutex, &mattr))) +#ifdef MDB_ROBUST_SUPPORTED + || (rc = pthread_mutexattr_setrobust(&mattr, PTHREAD_MUTEX_ROBUST)) +#endif + || (rc = pthread_mutex_init(env->me_txns->mti_rmutex, &mattr)) + || (rc = pthread_mutex_init(env->me_txns->mti_wmutex, &mattr))) goto fail; pthread_mutexattr_destroy(&mattr); #endif /* _WIN32 || MDB_USE_POSIX_SEM */ @@ -4349,8 +4605,8 @@ fail: * environment and re-opening it with the new flags. */ #define CHANGEABLE (MDB_NOSYNC|MDB_NOMETASYNC|MDB_MAPASYNC|MDB_NOMEMINIT) -#define CHANGELESS (MDB_FIXEDMAP|MDB_NOSUBDIR|MDB_RDONLY|MDB_WRITEMAP| \ - MDB_NOTLS|MDB_NOLOCK|MDB_NORDAHEAD) +#define CHANGELESS (MDB_FIXEDMAP|MDB_NOSUBDIR|MDB_RDONLY| \ + MDB_WRITEMAP|MDB_NOTLS|MDB_NOLOCK|MDB_NORDAHEAD) #if VALID_FLAGS & PERSISTENT_FLAGS & (CHANGEABLE|CHANGELESS) # error "Persistent DB flags & env flags overlap, but both go in mm_flags" @@ -4406,6 +4662,7 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode rc = ENOMEM; goto leave; } + env->me_dbxs[FREE_DBI].md_cmp = mdb_cmp_long; /* aligned MDB_INTEGERKEY */ /* For RDONLY, get lockfile after we know datafile exists */ if (!(flags & (MDB_RDONLY|MDB_NOLOCK))) { @@ -4471,9 +4728,24 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode if (rc) goto leave; } - if (!((flags & MDB_RDONLY) || - (env->me_pbuf = calloc(1, env->me_psize)))) - rc = ENOMEM; + if (!(flags & MDB_RDONLY)) { + MDB_txn *txn; + int tsize = sizeof(MDB_txn), size = tsize + env->me_maxdbs * + (sizeof(MDB_db)+sizeof(MDB_cursor *)+sizeof(unsigned int)+1); + if ((env->me_pbuf = calloc(1, env->me_psize)) && + (txn = calloc(1, size))) + { + txn->mt_dbs = (MDB_db *)((char *)txn + tsize); + txn->mt_cursors = (MDB_cursor **)(txn->mt_dbs + env->me_maxdbs); + txn->mt_dbiseqs = (unsigned int *)(txn->mt_cursors + env->me_maxdbs); + txn->mt_dbflags = (unsigned char *)(txn->mt_dbiseqs + env->me_maxdbs); + txn->mt_env = env; + txn->mt_dbxs = env->me_dbxs; + env->me_txn0 = txn; + } else { + rc = ENOMEM; + } + } } leave: @@ -4494,15 +4766,18 @@ mdb_env_close0(MDB_env *env, int excl) return; /* Doing this here since me_dbxs may not exist during mdb_env_close */ - for (i = env->me_maxdbs; --i > MAIN_DBI; ) - free(env->me_dbxs[i].md_name.mv_data); + if (env->me_dbxs) { + for (i = env->me_maxdbs; --i > MAIN_DBI; ) + free(env->me_dbxs[i].md_name.mv_data); + free(env->me_dbxs); + } free(env->me_pbuf); free(env->me_dbiseqs); free(env->me_dbflags); - free(env->me_dbxs); free(env->me_path); free(env->me_dirty_list); + free(env->me_txn0); mdb_midl_free(env->me_free_pgs); if (env->me_flags & MDB_ENV_TXKEY) { @@ -4529,8 +4804,12 @@ mdb_env_close0(MDB_env *env, int excl) MDB_PID_T pid = env->me_pid; /* Clearing readers is done in this function because * me_txkey with its destructor must be disabled first. + * + * We skip the the reader mutex, so we touch only + * data owned by this process (me_close_readers and + * our readers), and clear each reader atomically. */ - for (i = env->me_numreaders; --i >= 0; ) + for (i = env->me_close_readers; --i >= 0; ) if (env->me_txns->mti_readers[i].mr_pid == pid) env->me_txns->mti_readers[i].mr_pid = 0; #ifdef _WIN32 @@ -4574,7 +4853,6 @@ mdb_env_close0(MDB_env *env, int excl) env->me_flags &= ~(MDB_ENV_ACTIVE|MDB_ENV_TXKEY); } - void ESECT mdb_env_close(MDB_env *env) { @@ -4602,7 +4880,11 @@ mdb_cmp_long(const MDB_val *a, const MDB_val *b) *(size_t *)a->mv_data > *(size_t *)b->mv_data; } -/** Compare two items pointing at aligned unsigned int's */ +/** Compare two items pointing at aligned unsigned int's. + * + * This is also set as #MDB_INTEGERDUP|#MDB_DUPFIXED's #MDB_dbx.%md_dcmp, + * but #mdb_cmp_clong() is called instead if the data type is size_t. + */ static int mdb_cmp_int(const MDB_val *a, const MDB_val *b) { @@ -4640,13 +4922,6 @@ mdb_cmp_cint(const MDB_val *a, const MDB_val *b) #endif } -/** Compare two items pointing at size_t's of unknown alignment. */ -#ifdef MISALIGNED_OK -# define mdb_cmp_clong mdb_cmp_long -#else -# define mdb_cmp_clong mdb_cmp_cint -#endif - /** Compare two items lexically */ static int mdb_cmp_memn(const MDB_val *a, const MDB_val *b) @@ -4809,15 +5084,12 @@ static void mdb_cursor_pop(MDB_cursor *mc) { if (mc->mc_snum) { -#if MDB_DEBUG - MDB_page *top = mc->mc_pg[mc->mc_top]; -#endif + DPRINTF(("popping page %"Z"u off db %d cursor %p", + mc->mc_pg[mc->mc_top]->mp_pgno, DDBI(mc), (void *) mc)); + mc->mc_snum--; if (mc->mc_snum) mc->mc_top--; - - DPRINTF(("popped page %"Z"u off db %d cursor %p", top->mp_pgno, - DDBI(mc), (void *) mc)); } } @@ -4854,7 +5126,7 @@ mdb_page_get(MDB_txn *txn, pgno_t pgno, MDB_page **ret, int *lvl) MDB_page *p = NULL; int level; - if (!((txn->mt_flags & MDB_TXN_RDONLY) | (env->me_flags & MDB_WRITEMAP))) { + if (! (txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_WRITEMAP))) { MDB_txn *tx2 = txn; level = 1; do { @@ -5032,6 +5304,8 @@ mdb_page_search(MDB_cursor *mc, MDB_val *key, int flags) &mc->mc_dbx->md_name, &exact); if (!exact) return MDB_NOTFOUND; + if ((leaf->mn_flags & (F_DUPDATA|F_SUBDATA)) != F_SUBDATA) + return MDB_INCOMPATIBLE; /* not a named DB */ rc = mdb_node_read(mc->mc_txn, leaf, &data); if (rc) return rc; @@ -5194,7 +5468,7 @@ mdb_get(MDB_txn *txn, MDB_dbi dbi, DPRINTF(("===> get db %u key [%s]", dbi, DKEY(key))); - if (!key || !data || dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) + if (!key || !data || !TXN_DBI_EXIST(txn, dbi, DB_USRVALID)) return EINVAL; if (txn->mt_flags & MDB_TXN_ERROR) @@ -5366,11 +5640,11 @@ mdb_cursor_prev(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op) } return rc; } - } else { - mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); - if (op == MDB_PREV_DUP) - return MDB_NOTFOUND; } + } else { + mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); + if (op == MDB_PREV_DUP) + return MDB_NOTFOUND; } } @@ -5543,8 +5817,10 @@ set2: if (leaf == NULL) { DPUTS("===> inexact leaf not found, goto sibling"); - if ((rc = mdb_cursor_sibling(mc, 1)) != MDB_SUCCESS) + if ((rc = mdb_cursor_sibling(mc, 1)) != MDB_SUCCESS) { + mc->mc_flags |= C_EOF; return rc; /* no entries matched */ + } mp = mc->mc_pg[mc->mc_top]; mdb_cassert(mc, IS_LEAF(mp)); leaf = NODEPTR(mp, 0); @@ -5582,15 +5858,21 @@ set1: return rc; } } else if (op == MDB_GET_BOTH || op == MDB_GET_BOTH_RANGE) { - MDB_val d2; - if ((rc = mdb_node_read(mc->mc_txn, leaf, &d2)) != MDB_SUCCESS) + MDB_val olddata; + MDB_cmp_func *dcmp; + if ((rc = mdb_node_read(mc->mc_txn, leaf, &olddata)) != MDB_SUCCESS) return rc; - rc = mc->mc_dbx->md_dcmp(data, &d2); + dcmp = mc->mc_dbx->md_dcmp; +#if UINT_MAX < SIZE_MAX + if (dcmp == mdb_cmp_int && olddata.mv_size == sizeof(size_t)) + dcmp = mdb_cmp_clong; +#endif + rc = dcmp(data, &olddata); if (rc) { if (op == MDB_GET_BOTH || rc > 0) return MDB_NOTFOUND; rc = 0; - *data = d2; + *data = olddata; } } else { @@ -5844,8 +6126,7 @@ fetchm: MDB_node *leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); if (!F_ISSET(leaf->mn_flags, F_DUPDATA)) { MDB_GET_KEY(leaf, key); - if (data) - rc = mdb_node_read(mc->mc_txn, leaf, data); + rc = mdb_node_read(mc->mc_txn, leaf, data); break; } } @@ -5910,10 +6191,9 @@ int mdb_cursor_put(MDB_cursor *mc, MDB_val *key, MDB_val *data, unsigned int flags) { - enum { MDB_NO_ROOT = MDB_LAST_ERRCODE+10 }; /* internal code */ MDB_env *env; MDB_node *leaf = NULL; - MDB_page *fp, *mp; + MDB_page *fp, *mp, *sub_root = NULL; uint16_t fp_flags; MDB_val xdata, *rdata, dkey, olddata; MDB_db dummy; @@ -6101,16 +6381,17 @@ more: /* Was a single item before, must convert now */ if (!F_ISSET(leaf->mn_flags, F_DUPDATA)) { + MDB_cmp_func *dcmp; /* Just overwrite the current item */ if (flags == MDB_CURRENT) goto current; - + dcmp = mc->mc_dbx->md_dcmp; #if UINT_MAX < SIZE_MAX - if (mc->mc_dbx->md_dcmp == mdb_cmp_int && olddata.mv_size == sizeof(size_t)) - mc->mc_dbx->md_dcmp = mdb_cmp_clong; + if (dcmp == mdb_cmp_int && olddata.mv_size == sizeof(size_t)) + dcmp = mdb_cmp_clong; #endif /* does data match? */ - if (!mc->mc_dbx->md_dcmp(data, &olddata)) { + if (!dcmp(data, &olddata)) { if (flags & MDB_NODUPDATA) return MDB_KEYEXIST; /* overwrite it */ @@ -6192,6 +6473,7 @@ prep_subDB: offset = env->me_psize - olddata.mv_size; flags |= F_DUPDATA|F_SUBDATA; dummy.md_root = mp->mp_pgno; + sub_root = mp; } if (mp != fp) { mp->mp_flags = fp_flags | P_DIRTY; @@ -6216,6 +6498,9 @@ prep_subDB: goto new_sub; } current: + /* LMDB passes F_SUBDATA in 'flags' to write a DB record */ + if ((leaf->mn_flags ^ flags) & F_SUBDATA) + return MDB_INCOMPATIBLE; /* overflow page overwrites need special handling */ if (F_ISSET(leaf->mn_flags, F_BIGDATA)) { MDB_page *omp; @@ -6335,7 +6620,7 @@ new_sub: * DB are all zero size. */ if (do_sub) { - int xflags; + int xflags, new_dupdata; size_t ecount; put_sub: xdata.mv_size = 0; @@ -6348,27 +6633,37 @@ put_sub: xflags = (flags & MDB_NODUPDATA) ? MDB_NOOVERWRITE|MDB_NOSPILL : MDB_NOSPILL; } + if (sub_root) + mc->mc_xcursor->mx_cursor.mc_pg[0] = sub_root; + new_dupdata = (int)dkey.mv_size; /* converted, write the original data first */ if (dkey.mv_size) { rc = mdb_cursor_put(&mc->mc_xcursor->mx_cursor, &dkey, &xdata, xflags); if (rc) goto bad_sub; - { - /* Adjust other cursors pointing to mp */ - MDB_cursor *m2; - unsigned i = mc->mc_top; - MDB_page *mp = mc->mc_pg[i]; - - for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2=m2->mc_next) { - if (m2 == mc || m2->mc_snum < mc->mc_snum) continue; - if (!(m2->mc_flags & C_INITIALIZED)) continue; - if (m2->mc_pg[i] == mp && m2->mc_ki[i] == mc->mc_ki[i]) { - mdb_xcursor_init1(m2, leaf); + /* we've done our job */ + dkey.mv_size = 0; + } + if (!(leaf->mn_flags & F_SUBDATA) || sub_root) { + /* Adjust other cursors pointing to mp */ + MDB_cursor *m2; + MDB_xcursor *mx = mc->mc_xcursor; + unsigned i = mc->mc_top; + MDB_page *mp = mc->mc_pg[i]; + + for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2=m2->mc_next) { + if (m2 == mc || m2->mc_snum < mc->mc_snum) continue; + if (!(m2->mc_flags & C_INITIALIZED)) continue; + if (m2->mc_pg[i] == mp) { + if (m2->mc_ki[i] == mc->mc_ki[i]) { + mdb_xcursor_init2(m2, mx, new_dupdata); + } else if (!insert_key) { + MDB_node *n2 = NODEPTR(mp, m2->mc_ki[i]); + if (!(n2->mn_flags & F_SUBDATA)) + m2->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(n2); } } } - /* we've done our job */ - dkey.mv_size = 0; } ecount = mc->mc_xcursor->mx_db.md_entries; if (flags & MDB_APPENDDUP) @@ -6464,12 +6759,19 @@ mdb_cursor_del(MDB_cursor *mc, unsigned int flags) mdb_node_shrink(mp, mc->mc_ki[mc->mc_top]); leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); mc->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(leaf); - /* fix other sub-DB cursors pointed at this fake page */ + /* fix other sub-DB cursors pointed at fake pages on this page */ for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2=m2->mc_next) { if (m2 == mc || m2->mc_snum < mc->mc_snum) continue; - if (m2->mc_pg[mc->mc_top] == mp && - m2->mc_ki[mc->mc_top] == mc->mc_ki[mc->mc_top]) - m2->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(leaf); + if (!(m2->mc_flags & C_INITIALIZED)) continue; + if (m2->mc_pg[mc->mc_top] == mp) { + if (m2->mc_ki[mc->mc_top] == mc->mc_ki[mc->mc_top]) { + m2->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(leaf); + } else { + MDB_node *n2 = NODEPTR(mp, m2->mc_ki[mc->mc_top]); + if (!(n2->mn_flags & F_SUBDATA)) + m2->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(n2); + } + } } } mc->mc_db->md_entries--; @@ -6486,6 +6788,11 @@ mdb_cursor_del(MDB_cursor *mc, unsigned int flags) goto fail; } } + /* LMDB passes F_SUBDATA in 'flags' to delete a DB record */ + else if ((leaf->mn_flags ^ flags) & F_SUBDATA) { + rc = MDB_INCOMPATIBLE; + goto fail; + } /* add overflow pages to free list */ if (F_ISSET(leaf->mn_flags, F_BIGDATA)) { @@ -6796,45 +7103,38 @@ mdb_node_shrink(MDB_page *mp, indx_t indx) MDB_node *node; MDB_page *sp, *xp; char *base; - int nsize, delta; - indx_t i, numkeys, ptr; + indx_t delta, nsize, len, ptr; + int i; node = NODEPTR(mp, indx); sp = (MDB_page *)NODEDATA(node); delta = SIZELEFT(sp); - xp = (MDB_page *)((char *)sp + delta); + nsize = NODEDSZ(node) - delta; - /* shift subpage upward */ + /* Prepare to shift upward, set len = length(subpage part to shift) */ if (IS_LEAF2(sp)) { - nsize = NUMKEYS(sp) * sp->mp_pad; + len = nsize; if (nsize & 1) return; /* do not make the node uneven-sized */ - memmove(METADATA(xp), METADATA(sp), nsize); } else { - int i; - numkeys = NUMKEYS(sp); - for (i=numkeys-1; i>=0; i--) + xp = (MDB_page *)((char *)sp + delta); /* destination subpage */ + for (i = NUMKEYS(sp); --i >= 0; ) xp->mp_ptrs[i] = sp->mp_ptrs[i] - delta; + len = PAGEHDRSZ; } - xp->mp_upper = sp->mp_lower; - xp->mp_lower = sp->mp_lower; - xp->mp_flags = sp->mp_flags; - xp->mp_pad = sp->mp_pad; - COPY_PGNO(xp->mp_pgno, mp->mp_pgno); - - nsize = NODEDSZ(node) - delta; + sp->mp_upper = sp->mp_lower; + COPY_PGNO(sp->mp_pgno, mp->mp_pgno); SETDSZ(node, nsize); - /* shift lower nodes upward */ + /* Shift upward */ + base = (char *)mp + mp->mp_upper + PAGEBASE; + memmove(base + delta, base, (char *)sp + len - base); + ptr = mp->mp_ptrs[indx]; - numkeys = NUMKEYS(mp); - for (i = 0; i < numkeys; i++) { + for (i = NUMKEYS(mp); --i >= 0; ) { if (mp->mp_ptrs[i] <= ptr) mp->mp_ptrs[i] += delta; } - - base = (char *)mp + mp->mp_upper + PAGEBASE; - memmove(base + delta, base, ptr - mp->mp_upper + NODESIZE + NODEKSZ(node)); mp->mp_upper += delta; } @@ -6887,7 +7187,7 @@ mdb_xcursor_init1(MDB_cursor *mc, MDB_node *node) mx->mx_cursor.mc_flags = C_SUB; } else { MDB_page *fp = NODEDATA(node); - mx->mx_db.md_pad = mc->mc_pg[mc->mc_top]->mp_pad; + mx->mx_db.md_pad = 0; mx->mx_db.md_flags = 0; mx->mx_db.md_depth = 1; mx->mx_db.md_branch_pages = 0; @@ -6909,13 +7209,45 @@ mdb_xcursor_init1(MDB_cursor *mc, MDB_node *node) } DPRINTF(("Sub-db -%u root page %"Z"u", mx->mx_cursor.mc_dbi, mx->mx_db.md_root)); - mx->mx_dbflag = DB_VALID|DB_DIRTY; /* DB_DIRTY guides mdb_cursor_touch */ + mx->mx_dbflag = DB_VALID|DB_USRVALID|DB_DIRTY; /* DB_DIRTY guides mdb_cursor_touch */ #if UINT_MAX < SIZE_MAX if (mx->mx_dbx.md_cmp == mdb_cmp_int && mx->mx_db.md_pad == sizeof(size_t)) mx->mx_dbx.md_cmp = mdb_cmp_clong; #endif } + +/** Fixup a sorted-dups cursor due to underlying update. + * Sets up some fields that depend on the data from the main cursor. + * Almost the same as init1, but skips initialization steps if the + * xcursor had already been used. + * @param[in] mc The main cursor whose sorted-dups cursor is to be fixed up. + * @param[in] src_mx The xcursor of an up-to-date cursor. + * @param[in] new_dupdata True if converting from a non-#F_DUPDATA item. + */ +static void +mdb_xcursor_init2(MDB_cursor *mc, MDB_xcursor *src_mx, int new_dupdata) +{ + MDB_xcursor *mx = mc->mc_xcursor; + + if (new_dupdata) { + mx->mx_cursor.mc_snum = 1; + mx->mx_cursor.mc_top = 0; + mx->mx_cursor.mc_flags |= C_INITIALIZED; + mx->mx_cursor.mc_ki[0] = 0; + mx->mx_dbflag = DB_VALID|DB_USRVALID|DB_DIRTY; /* DB_DIRTY guides mdb_cursor_touch */ +#if UINT_MAX < SIZE_MAX + mx->mx_dbx.md_cmp = src_mx->mx_dbx.md_cmp; +#endif + } else if (!(mx->mx_cursor.mc_flags & C_INITIALIZED)) { + return; + } + mx->mx_db = src_mx->mx_db; + mx->mx_cursor.mc_pg[0] = src_mx->mx_cursor.mc_pg[0]; + DPRINTF(("Sub-db -%u root page %"Z"u", mx->mx_cursor.mc_dbi, + mx->mx_db.md_root)); +} + /** Initialize a cursor for a given transaction and database. */ static void mdb_cursor_init(MDB_cursor *mc, MDB_txn *txn, MDB_dbi dbi, MDB_xcursor *mx) @@ -6930,6 +7262,7 @@ mdb_cursor_init(MDB_cursor *mc, MDB_txn *txn, MDB_dbi dbi, MDB_xcursor *mx) mc->mc_snum = 0; mc->mc_top = 0; mc->mc_pg[0] = 0; + mc->mc_ki[0] = 0; mc->mc_flags = 0; if (txn->mt_dbs[dbi].md_flags & MDB_DUPSORT) { mdb_tassert(txn, mx != NULL); @@ -6949,7 +7282,7 @@ mdb_cursor_open(MDB_txn *txn, MDB_dbi dbi, MDB_cursor **ret) MDB_cursor *mc; size_t size = sizeof(MDB_cursor); - if (!ret || !TXN_DBI_EXIST(txn, dbi)) + if (!ret || !TXN_DBI_EXIST(txn, dbi, DB_VALID)) return EINVAL; if (txn->mt_flags & MDB_TXN_ERROR) @@ -6981,7 +7314,7 @@ mdb_cursor_open(MDB_txn *txn, MDB_dbi dbi, MDB_cursor **ret) int mdb_cursor_renew(MDB_txn *txn, MDB_cursor *mc) { - if (!mc || !TXN_DBI_EXIST(txn, mc->mc_dbi)) + if (!mc || !TXN_DBI_EXIST(txn, mc->mc_dbi, DB_VALID)) return EINVAL; if ((mc->mc_flags & C_UNTRACK) || txn->mt_cursors) @@ -7233,8 +7566,22 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst) /* Adjust other cursors pointing to mp */ MDB_cursor *m2, *m3; MDB_dbi dbi = csrc->mc_dbi; - MDB_page *mp = csrc->mc_pg[csrc->mc_top]; + MDB_page *mp; + mp = cdst->mc_pg[csrc->mc_top]; + for (m2 = csrc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { + if (csrc->mc_flags & C_SUB) + m3 = &m2->mc_xcursor->mx_cursor; + else + m3 = m2; + if (m3 == cdst) continue; + if (m3->mc_pg[csrc->mc_top] == mp && m3->mc_ki[csrc->mc_top] >= + cdst->mc_ki[csrc->mc_top]) { + m3->mc_ki[csrc->mc_top]++; + } + } + + mp = csrc->mc_pg[csrc->mc_top]; for (m2 = csrc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { if (csrc->mc_flags & C_SUB) m3 = &m2->mc_xcursor->mx_cursor; @@ -7303,7 +7650,7 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst) cdst->mc_ki[cdst->mc_top] = 0; rc = mdb_update_key(cdst, &nullkey); cdst->mc_ki[cdst->mc_top] = ix; - mdb_cassert(csrc, rc == MDB_SUCCESS); + mdb_cassert(cdst, rc == MDB_SUCCESS); } } @@ -7436,9 +7783,9 @@ mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst) uint16_t depth = cdst->mc_db->md_depth; mdb_cursor_pop(cdst); rc = mdb_rebalance(cdst); - /* Did the tree shrink? */ - if (depth > cdst->mc_db->md_depth) - snum--; + /* Did the tree height change? */ + if (depth != cdst->mc_db->md_depth) + snum += cdst->mc_db->md_depth - depth; cdst->mc_snum = snum; cdst->mc_top = snum-1; } @@ -7478,17 +7825,23 @@ mdb_rebalance(MDB_cursor *mc) { MDB_node *node; int rc; - unsigned int ptop, minkeys; + unsigned int ptop, minkeys, thresh; MDB_cursor mn; indx_t oldki; - minkeys = 1 + (IS_BRANCH(mc->mc_pg[mc->mc_top])); + if (IS_BRANCH(mc->mc_pg[mc->mc_top])) { + minkeys = 2; + thresh = 1; + } else { + minkeys = 1; + thresh = FILL_THRESHOLD; + } DPRINTF(("rebalancing %s page %"Z"u (has %u keys, %.1f%% full)", IS_LEAF(mc->mc_pg[mc->mc_top]) ? "leaf" : "branch", mdb_dbg_pgno(mc->mc_pg[mc->mc_top]), NUMKEYS(mc->mc_pg[mc->mc_top]), (float)PAGEFILL(mc->mc_txn->mt_env, mc->mc_pg[mc->mc_top]) / 10)); - if (PAGEFILL(mc->mc_txn->mt_env, mc->mc_pg[mc->mc_top]) >= FILL_THRESHOLD && + if (PAGEFILL(mc->mc_txn->mt_env, mc->mc_pg[mc->mc_top]) >= thresh && NUMKEYS(mc->mc_pg[mc->mc_top]) >= minkeys) { DPRINTF(("no need to rebalance page %"Z"u, above fill threshold", mdb_dbg_pgno(mc->mc_pg[mc->mc_top]))); @@ -7559,12 +7912,12 @@ mdb_rebalance(MDB_cursor *mc) m3 = m2; if (m3 == mc || m3->mc_snum < mc->mc_snum) continue; if (m3->mc_pg[0] == mp) { - m3->mc_snum--; - m3->mc_top--; for (i=0; imc_snum; i++) { m3->mc_pg[i] = m3->mc_pg[i+1]; m3->mc_ki[i] = m3->mc_ki[i+1]; } + m3->mc_snum--; + m3->mc_top--; } } } @@ -7622,19 +7975,32 @@ mdb_rebalance(MDB_cursor *mc) * move one key from it. Otherwise we should try to merge them. * (A branch page must never have less than 2 keys.) */ - minkeys = 1 + (IS_BRANCH(mn.mc_pg[mn.mc_top])); - if (PAGEFILL(mc->mc_txn->mt_env, mn.mc_pg[mn.mc_top]) >= FILL_THRESHOLD && NUMKEYS(mn.mc_pg[mn.mc_top]) > minkeys) { + if (PAGEFILL(mc->mc_txn->mt_env, mn.mc_pg[mn.mc_top]) >= thresh && NUMKEYS(mn.mc_pg[mn.mc_top]) > minkeys) { rc = mdb_node_move(&mn, mc); - if (mc->mc_ki[ptop]) { + if (mc->mc_ki[mc->mc_top-1]) { oldki++; } } else { if (mc->mc_ki[ptop] == 0) { rc = mdb_page_merge(&mn, mc); } else { + MDB_cursor dummy; oldki += NUMKEYS(mn.mc_pg[mn.mc_top]); mn.mc_ki[mn.mc_top] += mc->mc_ki[mn.mc_top] + 1; + /* We want mdb_rebalance to find mn when doing fixups */ + if (mc->mc_flags & C_SUB) { + dummy.mc_next = mc->mc_txn->mt_cursors[mc->mc_dbi]; + mc->mc_txn->mt_cursors[mc->mc_dbi] = &dummy; + dummy.mc_xcursor = (MDB_xcursor *)&mn; + } else { + mn.mc_next = mc->mc_txn->mt_cursors[mc->mc_dbi]; + mc->mc_txn->mt_cursors[mc->mc_dbi] = &mn; + } rc = mdb_page_merge(mc, &mn); + if (mc->mc_flags & C_SUB) + mc->mc_txn->mt_cursors[mc->mc_dbi] = dummy.mc_next; + else + mc->mc_txn->mt_cursors[mc->mc_dbi] = mn.mc_next; mdb_cursor_copy(&mn, mc); } mc->mc_flags &= ~C_EOF; @@ -7651,30 +8017,16 @@ mdb_cursor_del0(MDB_cursor *mc) MDB_page *mp; indx_t ki; unsigned int nkeys; + MDB_cursor *m2, *m3; + MDB_dbi dbi = mc->mc_dbi; ki = mc->mc_ki[mc->mc_top]; + mp = mc->mc_pg[mc->mc_top]; mdb_node_del(mc, mc->mc_db->md_pad); mc->mc_db->md_entries--; - rc = mdb_rebalance(mc); - - if (rc == MDB_SUCCESS) { - MDB_cursor *m2, *m3; - MDB_dbi dbi = mc->mc_dbi; - - mp = mc->mc_pg[mc->mc_top]; - nkeys = NUMKEYS(mp); - - /* if mc points past last node in page, find next sibling */ - if (mc->mc_ki[mc->mc_top] >= nkeys) { - rc = mdb_cursor_sibling(mc, 1); - if (rc == MDB_NOTFOUND) { - mc->mc_flags |= C_EOF; - rc = MDB_SUCCESS; - } - } - + { /* Adjust other cursors pointing to mp */ - for (m2 = mc->mc_txn->mt_cursors[dbi]; !rc && m2; m2=m2->mc_next) { + for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { m3 = (mc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2; if (! (m2->mc_flags & m3->mc_flags & C_INITIALIZED)) continue; @@ -7688,6 +8040,31 @@ mdb_cursor_del0(MDB_cursor *mc) else if (mc->mc_db->md_flags & MDB_DUPSORT) m3->mc_xcursor->mx_cursor.mc_flags |= C_EOF; } + } + } + } + rc = mdb_rebalance(mc); + + if (rc == MDB_SUCCESS) { + /* DB is totally empty now, just bail out. + * Other cursors adjustments were already done + * by mdb_rebalance and aren't needed here. + */ + if (!mc->mc_snum) + return rc; + + mp = mc->mc_pg[mc->mc_top]; + nkeys = NUMKEYS(mp); + + /* Adjust other cursors pointing to mp */ + for (m2 = mc->mc_txn->mt_cursors[dbi]; !rc && m2; m2=m2->mc_next) { + m3 = (mc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2; + if (! (m2->mc_flags & m3->mc_flags & C_INITIALIZED)) + continue; + if (m3->mc_snum < mc->mc_snum) + continue; + if (m3->mc_pg[mc->mc_top] == mp) { + /* if m3 points past last node in page, find next sibling */ if (m3->mc_ki[mc->mc_top] >= nkeys) { rc = mdb_cursor_sibling(m3, 1); if (rc == MDB_NOTFOUND) { @@ -7709,7 +8086,7 @@ int mdb_del(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data) { - if (!key || dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) + if (!key || !TXN_DBI_EXIST(txn, dbi, DB_USRVALID)) return EINVAL; if (txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_ERROR)) @@ -7805,6 +8182,7 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno /* Create a right sibling. */ if ((rc = mdb_page_new(mc, mp->mp_flags, 1, &rp))) return rc; + rp->mp_pad = mp->mp_pad; DPRINTF(("new right sibling: page %"Z"u", rp->mp_pgno)); if (mc->mc_snum < 2) { @@ -7817,8 +8195,7 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno mc->mc_ki[0] = 0; mc->mc_db->md_root = pp->mp_pgno; DPRINTF(("root split! new root = %"Z"u", pp->mp_pgno)); - mc->mc_db->md_depth++; - new_root = 1; + new_root = mc->mc_db->md_depth++; /* Add left (implicit) pointer. */ if ((rc = mdb_node_add(mc, 0, NULL, NULL, mp->mp_pgno, 0)) != MDB_SUCCESS) { @@ -7939,7 +8316,7 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno psize = 0; if (newindx <= split_indx || newindx >= nkeys) { i = 0; j = 1; - k = newindx >= nkeys ? nkeys : split_indx+2; + k = newindx >= nkeys ? nkeys : split_indx+1+IS_LEAF(mp); } else { i = nkeys; j = -1; k = split_indx-1; @@ -8131,7 +8508,7 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno if (new_root) { int k; /* root split */ - for (k=m3->mc_top; k>=0; k--) { + for (k=new_root; k>=0; k--) { m3->mc_ki[k+1] = m3->mc_ki[k]; m3->mc_pg[k+1] = m3->mc_pg[k]; } @@ -8175,10 +8552,10 @@ mdb_put(MDB_txn *txn, MDB_dbi dbi, MDB_cursor mc; MDB_xcursor mx; - if (!key || !data || dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) + if (!key || !data || !TXN_DBI_EXIST(txn, dbi, DB_USRVALID)) return EINVAL; - if ((flags & (MDB_NOOVERWRITE|MDB_NODUPDATA|MDB_RESERVE|MDB_APPEND|MDB_APPENDDUP)) != flags) + if (flags & ~(MDB_NOOVERWRITE|MDB_NODUPDATA|MDB_RESERVE|MDB_APPEND|MDB_APPENDDUP)) return EINVAL; mdb_cursor_init(&mc, txn, dbi, &mx); @@ -8208,7 +8585,7 @@ typedef struct mdb_copy { } mdb_copy; /** Dedicated writer thread for compacting copy. */ -static THREAD_RET ESECT +static THREAD_RET ESECT CALL_CONV mdb_env_copythr(void *arg) { mdb_copy *my = arg; @@ -8469,9 +8846,15 @@ mdb_env_copyfd1(MDB_env *env, HANDLE fd) #else pthread_mutex_init(&my.mc_mutex, NULL); pthread_cond_init(&my.mc_cond, NULL); +#ifdef HAVE_MEMALIGN + my.mc_wbuf[0] = memalign(env->me_os_psize, MDB_WBUF*2); + if (my.mc_wbuf[0] == NULL) + return errno; +#else rc = posix_memalign((void **)&my.mc_wbuf[0], env->me_os_psize, MDB_WBUF*2); if (rc) return rc; +#endif #endif memset(my.mc_wbuf[0], 0, MDB_WBUF*2); my.mc_wbuf[1] = my.mc_wbuf[0] + MDB_WBUF; @@ -8522,8 +8905,12 @@ mdb_env_copyfd1(MDB_env *env, HANDLE fd) /* Set metapage 1 */ mm->mm_last_pg = txn->mt_next_pgno - freecount - 1; mm->mm_dbs[1] = txn->mt_dbs[1]; - mm->mm_dbs[1].md_root = mm->mm_last_pg; - mm->mm_txnid = 1; + if (mm->mm_last_pg > 1) { + mm->mm_dbs[1].md_root = mm->mm_last_pg; + mm->mm_txnid = 1; + } else { + mm->mm_dbs[1].md_root = P_INVALID; + } } my.mc_wlen[0] = env->me_psize * 2; my.mc_txn = txn; @@ -8559,6 +8946,7 @@ static int ESECT mdb_env_copyfd0(MDB_env *env, HANDLE fd) { MDB_txn *txn = NULL; + mdb_mutexref_t wmutex = NULL; int rc; size_t wsize; char *ptr; @@ -8583,11 +8971,13 @@ mdb_env_copyfd0(MDB_env *env, HANDLE fd) mdb_txn_reset0(txn, "reset-stage1"); /* Temporarily block writers until we snapshot the meta pages */ - LOCK_MUTEX_W(env); + wmutex = env->me_wmutex; + if (LOCK_MUTEX(rc, env, wmutex)) + goto leave; rc = mdb_txn_renew0(txn); if (rc) { - UNLOCK_MUTEX_W(env); + UNLOCK_MUTEX(wmutex); goto leave; } } @@ -8611,28 +9001,20 @@ mdb_env_copyfd0(MDB_env *env, HANDLE fd) break; } } - if (env->me_txns) - UNLOCK_MUTEX_W(env); + if (wmutex) + UNLOCK_MUTEX(wmutex); if (rc) goto leave; w2 = txn->mt_next_pgno * env->me_psize; -#ifdef WIN32 { - LARGE_INTEGER fsize; - GetFileSizeEx(env->me_fd, &fsize); - if (w2 > fsize.QuadPart) - w2 = fsize.QuadPart; - } -#else - { - struct stat st; - fstat(env->me_fd, &st); - if (w2 > (size_t)st.st_size) - w2 = st.st_size; + size_t fsize = 0; + if ((rc = mdb_fsize(env->me_fd, &fsize))) + goto leave; + if (w2 > fsize) + w2 = fsize; } -#endif wsize = w2 - wsize; while (wsize > 0) { if (wsize > MAX_WRITE) @@ -8743,7 +9125,7 @@ mdb_env_copy(MDB_env *env, const char *path) int ESECT mdb_env_set_flags(MDB_env *env, unsigned int flag, int onoff) { - if ((flag & CHANGEABLE) != flag) + if (flag & ~CHANGEABLE) return EINVAL; if (onoff) env->me_flags |= flag; @@ -8758,7 +9140,7 @@ mdb_env_get_flags(MDB_env *env, unsigned int *arg) if (!env || !arg) return EINVAL; - *arg = env->me_flags; + *arg = env->me_flags & (CHANGEABLE|CHANGELESS); return MDB_SUCCESS; } @@ -8852,11 +9234,7 @@ mdb_env_info(MDB_env *env, MDB_envinfo *arg) arg->me_mapaddr = env->me_metas[toggle]->mm_address; arg->me_mapsize = env->me_mapsize; arg->me_maxreaders = env->me_maxreaders; - - /* me_numreaders may be zero if this process never used any readers. Use - * the shared numreader count if it exists. - */ - arg->me_numreaders = env->me_txns ? env->me_txns->mti_numreaders : env->me_numreaders; + arg->me_numreaders = env->me_txns ? env->me_txns->mti_numreaders : 0; arg->me_last_pgno = env->me_metas[toggle]->mm_last_pg; arg->me_last_txnid = env->me_metas[toggle]->mm_txnid; @@ -8891,15 +9269,12 @@ int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *db MDB_val key, data; MDB_dbi i; MDB_cursor mc; + MDB_db dummy; int rc, dbflag, exact; unsigned int unused = 0, seq; size_t len; - if (txn->mt_dbxs[FREE_DBI].md_cmp == NULL) { - mdb_default_cmp(txn, FREE_DBI); - } - - if ((flags & VALID_FLAGS) != flags) + if (flags & ~VALID_FLAGS) return EINVAL; if (txn->mt_flags & MDB_TXN_ERROR) return MDB_BAD_TXN; @@ -8947,7 +9322,7 @@ int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *db return (flags & MDB_CREATE) ? MDB_INCOMPATIBLE : MDB_NOTFOUND; /* Find the DB info */ - dbflag = DB_NEW|DB_VALID; + dbflag = DB_NEW|DB_VALID|DB_USRVALID; exact = 0; key.mv_size = len; key.mv_data = (void *)name; @@ -8956,11 +9331,10 @@ int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *db if (rc == MDB_SUCCESS) { /* make sure this is actually a DB */ MDB_node *node = NODEPTR(mc.mc_pg[mc.mc_top], mc.mc_ki[mc.mc_top]); - if (!(node->mn_flags & F_SUBDATA)) + if ((node->mn_flags & (F_DUPDATA|F_SUBDATA)) != F_SUBDATA) return MDB_INCOMPATIBLE; } else if (rc == MDB_NOTFOUND && (flags & MDB_CREATE)) { /* Create if requested */ - MDB_db dummy; data.mv_size = sizeof(MDB_db); data.mv_data = &dummy; memset(&dummy, 0, sizeof(dummy)); @@ -8996,7 +9370,7 @@ int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *db int mdb_stat(MDB_txn *txn, MDB_dbi dbi, MDB_stat *arg) { - if (!arg || !TXN_DBI_EXIST(txn, dbi)) + if (!arg || !TXN_DBI_EXIST(txn, dbi, DB_VALID)) return EINVAL; if (txn->mt_flags & MDB_TXN_ERROR) @@ -9030,7 +9404,7 @@ void mdb_dbi_close(MDB_env *env, MDB_dbi dbi) int mdb_dbi_flags(MDB_txn *txn, MDB_dbi dbi, unsigned int *flags) { /* We could return the flags for the FREE_DBI too but what's the point? */ - if (dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) + if (!TXN_DBI_EXIST(txn, dbi, DB_USRVALID)) return EINVAL; *flags = txn->mt_dbs[dbi].md_flags & PERSISTENT_FLAGS; return MDB_SUCCESS; @@ -9053,8 +9427,10 @@ mdb_drop0(MDB_cursor *mc, int subs) MDB_cursor mx; unsigned int i; - /* LEAF2 pages have no nodes, cannot have sub-DBs */ - if (IS_LEAF2(mc->mc_pg[mc->mc_top])) + /* DUPSORT sub-DBs have no ovpages/DBs. Omit scanning leaves. + * This also avoids any P_LEAF2 pages, which have no nodes. + */ + if (mc->mc_flags & C_SUB) mdb_cursor_pop(mc); mdb_cursor_copy(mc, &mx); @@ -9128,13 +9504,13 @@ int mdb_drop(MDB_txn *txn, MDB_dbi dbi, int del) MDB_cursor *mc, *m2; int rc; - if ((unsigned)del > 1 || dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) + if ((unsigned)del > 1 || !TXN_DBI_EXIST(txn, dbi, DB_USRVALID)) return EINVAL; if (F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)) return EACCES; - if (dbi > MAIN_DBI && TXN_DBI_CHANGED(txn, dbi)) + if (TXN_DBI_CHANGED(txn, dbi)) return MDB_BAD_DBI; rc = mdb_cursor_open(txn, dbi, &mc); @@ -9150,7 +9526,7 @@ int mdb_drop(MDB_txn *txn, MDB_dbi dbi, int del) /* Can't delete the main DB */ if (del && dbi > MAIN_DBI) { - rc = mdb_del0(txn, MAIN_DBI, &mc->mc_dbx->md_name, NULL, 0); + rc = mdb_del0(txn, MAIN_DBI, &mc->mc_dbx->md_name, NULL, F_SUBDATA); if (!rc) { txn->mt_dbflags[dbi] = DB_STALE; mdb_dbi_close(txn->mt_env, dbi); @@ -9176,7 +9552,7 @@ leave: int mdb_set_compare(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp) { - if (dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) + if (!TXN_DBI_EXIST(txn, dbi, DB_USRVALID)) return EINVAL; txn->mt_dbxs[dbi].md_cmp = cmp; @@ -9185,7 +9561,7 @@ int mdb_set_compare(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp) int mdb_set_dupsort(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp) { - if (dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) + if (!TXN_DBI_EXIST(txn, dbi, DB_USRVALID)) return EINVAL; txn->mt_dbxs[dbi].md_dcmp = cmp; @@ -9194,7 +9570,7 @@ int mdb_set_dupsort(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp) int mdb_set_relfunc(MDB_txn *txn, MDB_dbi dbi, MDB_rel_func *rel) { - if (dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) + if (!TXN_DBI_EXIST(txn, dbi, DB_USRVALID)) return EINVAL; txn->mt_dbxs[dbi].md_rel = rel; @@ -9203,7 +9579,7 @@ int mdb_set_relfunc(MDB_txn *txn, MDB_dbi dbi, MDB_rel_func *rel) int mdb_set_relctx(MDB_txn *txn, MDB_dbi dbi, void *ctx) { - if (dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) + if (!TXN_DBI_EXIST(txn, dbi, DB_USRVALID)) return EINVAL; txn->mt_dbxs[dbi].md_relctx = ctx; @@ -9297,17 +9673,22 @@ mdb_pid_insert(MDB_PID_T *ids, MDB_PID_T pid) int ESECT mdb_reader_check(MDB_env *env, int *dead) { - unsigned int i, j, rdrs; - MDB_reader *mr; - MDB_PID_T *pids, pid; - int count = 0; - if (!env) return EINVAL; if (dead) *dead = 0; - if (!env->me_txns) - return MDB_SUCCESS; + return env->me_txns ? mdb_reader_check0(env, 0, dead) : MDB_SUCCESS; +} + +/** As #mdb_reader_check(). rlocked = . */ +static int mdb_reader_check0(MDB_env *env, int rlocked, int *dead) +{ + mdb_mutexref_t rmutex = rlocked ? NULL : env->me_rmutex; + unsigned int i, j, rdrs; + MDB_reader *mr; + MDB_PID_T *pids, pid; + int rc = MDB_SUCCESS, count = 0; + rdrs = env->me_txns->mti_numreaders; pids = malloc((rdrs+1) * sizeof(MDB_PID_T)); if (!pids) @@ -9315,22 +9696,32 @@ mdb_reader_check(MDB_env *env, int *dead) pids[0] = 0; mr = env->me_txns->mti_readers; for (i=0; ime_pid) { - pid = mr[i].mr_pid; + pid = mr[i].mr_pid; + if (pid && pid != env->me_pid) { if (mdb_pid_insert(pids, pid) == 0) { if (!mdb_reader_pid(env, Pidcheck, pid)) { - LOCK_MUTEX_R(env); - /* Recheck, a new process may have reused pid */ - if (!mdb_reader_pid(env, Pidcheck, pid)) { - for (j=i; jme_rmutex); + if (!rlocked) { + /* Keep mti_txnid updated, otherwise next writer can + * overwrite data which latest meta page refers to. + */ + toggle = mdb_env_pick_meta(env); + env->me_txns->mti_txnid = env->me_metas[toggle]->mm_txnid; + /* env is hosed if the dead thread was ours */ + if (env->me_txn) { + env->me_flags |= MDB_FATAL_ERROR; + env->me_txn = NULL; + rc = MDB_PANIC; + } + } + DPRINTF(("%cmutex owner died, %s", (rlocked ? 'r' : 'w'), + (rc ? "this process' env is hosed" : "recovering"))); + rc2 = mdb_reader_check0(env, rlocked, NULL); + if (rc2 == 0) + rc2 = mdb_mutex_consistent(mutex); + if (rc || (rc = rc2)) { + DPRINTF(("LOCK_MUTEX recovery failed, %s", mdb_strerror(rc))); + UNLOCK_MUTEX(mutex); + } + } else { +#ifdef _WIN32 + rc = ErrCode(); +#endif + DPRINTF(("LOCK_MUTEX failed, %s", mdb_strerror(rc))); + } + + return rc; } +#endif /* MDB_ROBUST_SUPPORTED */ /** @} */