On-demand pagemapping for 32bit mode

Build with -DVL32 to enable.

diff --git a/libraries/liblmdb/Makefile b/libraries/liblmdb/Makefile
index 25c1095466fe4eafbfff107d66f6e370380b3079..095e65587fd22cfc521aee545f2ff1d4ec6436af 100644 (file)
--- a/libraries/liblmdb/Makefile
+++ b/libraries/liblmdb/Makefile
@@ -24,6 +24,7 @@ CFLAGS        = $(THREADS) $(OPT) $(W) $(XCFLAGS)
 LDLIBS =
 SOLIBS =
 prefix = /usr/local
+XCFLAGS = -DVL32
 
 ########################################################################
 

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 7ddb36e200b4c605ffe136fe2a91ab455179bd02..bb6460089f2269fe5edb254dfb047ff9d7307d1f 100644 (file)
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -988,6 +988,10 @@ struct MDB_txn {
        MDB_cursor      **mt_cursors;
        /** Array of flags for each DB */
        unsigned char   *mt_dbflags;
+#ifdef VL32
+       /** List of read-only pages */
+       MDB_ID2L        mt_rpages;
+#endif
        /**     Number of DB records in use. This number only ever increments;
         *      we don't decrement it when individual DB handles are closed.
         */
@@ -1090,6 +1094,9 @@ struct MDB_env {
        HANDLE          me_fd;          /**< The main data file */
        HANDLE          me_lfd;         /**< The lock file */
        HANDLE          me_mfd;                 /**< just for writing the meta pages */
+#if defined(VL32) && defined(_WIN32)
+       HANDLE          me_fmh;         /**< File Mapping handle */
+#endif
        /** Failed to update the meta page. Probably an I/O error. */
 #define        MDB_FATAL_ERROR 0x80000000U
        /** Some fields are initialized. */
@@ -2638,6 +2645,15 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret)
                DPRINTF(("calloc: %s", strerror(errno)));
                return ENOMEM;
        }
+#ifdef VL32
+       if (!parent) {
+               txn->mt_rpages = calloc(MDB_IDL_UM_SIZE, sizeof(MDB_ID2));
+               if (!txn->mt_rpages) {
+                       free(txn);
+                       return ENOMEM;
+               }
+       }
+#endif
        txn->mt_dbs = (MDB_db *) ((char *)txn + tsize);
        if (flags & MDB_RDONLY) {
                txn->mt_flags |= MDB_TXN_RDONLY;
@@ -2676,6 +2692,9 @@ ok:
                txn->mt_numdbs = parent->mt_numdbs;
                txn->mt_flags = parent->mt_flags;
                txn->mt_dbxs = parent->mt_dbxs;
+#ifdef VL32
+               txn->mt_rpages = parent->mt_rpages;
+#endif
                memcpy(txn->mt_dbs, parent->mt_dbs, txn->mt_numdbs * sizeof(MDB_db));
                /* Copy parent's mt_dbflags, but clear DB_NEW */
                for (i=0; i<txn->mt_numdbs; i++)
@@ -2799,6 +2818,22 @@ mdb_txn_reset0(MDB_txn *txn, const char *act)
                if (env->me_txns)
                        UNLOCK_MUTEX_W(env);
        }
+#ifdef VL32
+       {
+               unsigned i, n = txn->mt_rpages[0].mid;
+               for (i = 1; i <= n; i++) {
+#ifdef _WIN32
+                       UnmapViewOfFile(txn->mt_rpages[i].mptr);)
+#else
+                       MDB_page *mp = txn->mt_rpages[i].mptr;
+                       int size = txn->mt_env->me_psize;
+                       if (IS_OVERFLOW(mp)) size *= mp->mp_pages;
+                       munmap(mp, size);
+#endif
+               }
+       }
+       txn->mt_rpages[0].mid = 0;
+#endif
 }
 
 void
@@ -3714,13 +3749,29 @@ mdb_env_map(MDB_env *env, void *addr)
                sizehi, sizelo, NULL);
        if (!mh)
                return ErrCode();
+#ifdef VL32
+       msize = 2 * env->me_psize;
+#endif
        env->me_map = MapViewOfFileEx(mh, flags & MDB_WRITEMAP ?
                FILE_MAP_WRITE : FILE_MAP_READ,
                0, 0, msize, addr);
        rc = env->me_map ? 0 : ErrCode();
+#ifdef VL32
+       env->me_fmh = mh;
+#else
        CloseHandle(mh);
+#endif
        if (rc)
                return rc;
+#else
+#ifdef VL32
+       (void) flags;
+       env->me_map = mmap(addr, 2 * env->me_psize, PROT_READ, MAP_SHARED,
+               env->me_fd, 0);
+       if (env->me_map == MAP_FAILED) {
+               env->me_map = NULL;
+               return ErrCode();
+       }
 #else
        int prot = PROT_READ;
        if (flags & MDB_WRITEMAP) {
@@ -3754,6 +3805,7 @@ mdb_env_map(MDB_env *env, void *addr)
         */
        if (addr && env->me_map != addr)
                return EBUSY;   /* TODO: Make a new MDB_* error code? */
+#endif
 
        p = (MDB_page *)env->me_map;
        env->me_metas[0] = METADATA(p);
@@ -4405,6 +4457,17 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
        if (env->me_fd!=INVALID_HANDLE_VALUE || (flags & ~(CHANGEABLE|CHANGELESS)))
                return EINVAL;
 
+#ifdef VL32
+       if (flags & MDB_WRITEMAP) {
+               /* silently ignore WRITEMAP in 32 bit mode */
+               flags ^= MDB_WRITEMAP;
+       }
+       if (flags & MDB_FIXEDMAP) {
+               /* cannot support FIXEDMAP */
+               return EINVAL;
+       }
+#endif
+
        len = strlen(path);
        if (flags & MDB_NOSUBDIR) {
                rc = len + sizeof(LOCKSUFF) + len + 1;
@@ -4525,6 +4588,13 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
                                txn->mt_dbiseqs = (unsigned int *)(txn->mt_cursors + env->me_maxdbs);
                                txn->mt_dbflags = (unsigned char *)(txn->mt_dbiseqs + env->me_maxdbs);
                                txn->mt_env = env;
+#ifdef VL32
+                               txn->mt_rpages = calloc(MDB_IDL_UM_SIZE, sizeof(MDB_ID2));
+                               if (!txn->mt_rpages) {
+                                       free(txn);
+                                       rc = ENOMEM;
+                               }
+#endif
                                env->me_txn0 = txn;
                        } else {
                                rc = ENOMEM;
@@ -4576,7 +4646,11 @@ mdb_env_close0(MDB_env *env, int excl)
        }
 
        if (env->me_map) {
+#ifdef VL32
+               munmap(env->me_map, 2*env->me_psize);
+#else
                munmap(env->me_map, env->me_mapsize);
+#endif
        }
        if (env->me_mfd != env->me_fd && env->me_mfd != INVALID_HANDLE_VALUE)
                (void) close(env->me_mfd);
@@ -4943,7 +5017,56 @@ mdb_page_get(MDB_txn *txn, pgno_t pgno, MDB_page **ret, int *lvl)
 
        if (pgno < txn->mt_next_pgno) {
                level = 0;
+#ifdef VL32
+               {
+                       unsigned x = mdb_mid2l_search(txn->mt_rpages, pgno);
+                       if (x <= txn->mt_rpages[0].mid && txn->mt_rpages[x].mid == pgno) {
+                               p = txn->mt_rpages[x].mptr;
+                               goto done;
+                       }
+                       if (txn->mt_rpages[0].mid < MDB_IDL_UM_SIZE) {
+                               MDB_ID2 id2;
+                               size_t len = env->me_psize;
+                               int np;
+#ifdef _WIN32
+                               size_t off = pgno * env->me_psize;
+                               DWORD lo, hi;
+                               lo = off & 0xffffffff;
+                               hi = off >> 16 >> 16;
+                               p = MapViewOfFile(env->me_fmh, FILE_MAP_READ, hi, lo, len);
+                               if (p == NULL)
+                                       return ErrCode();
+                               if (IS_OVERFLOW(p)) {
+                                       np = p->mp_pages;
+                                       UnmapViewOfFile(p);
+                                       len *= np;
+                                       p = MapViewOfFile(env->me_fmh, FILE_MAP_READ, hi, lo, len);
+                                       if (p == NULL)
+                                               return ErrCode();
+                               }
+#else
+                               off_t off = pgno * env->me_psize;
+                               p = mmap(NULL, len, PROT_READ, MAP_SHARED, env->me_fd, off);
+                               if (p == MAP_FAILED)
+                                       return errno;
+                               if (IS_OVERFLOW(p)) {
+                                       np = p->mp_pages;
+                                       munmap(p, len);
+                                       len *= np;
+                                       p = mmap(NULL, len, PROT_READ, MAP_SHARED, env->me_fd, off);
+                                       if (p == MAP_FAILED)
+                                               return errno;
+                               }
+#endif
+                               id2.mid = pgno;
+                               id2.mptr = p;
+                               mdb_mid2l_insert(txn->mt_rpages, &id2);
+                               goto done;
+                       }
+               }
+#else
                p = (MDB_page *)(env->me_map + env->me_psize * pgno);
+#endif
        } else {
                DPRINTF(("page %"Z"u not found", pgno));
                txn->mt_flags |= MDB_TXN_ERROR;

diff --git a/libraries/liblmdb/midl.h b/libraries/liblmdb/midl.h
index a7f25026ce536eddb17c86be621ddfc8f9ee43f7..f13a547bc36c8b39ce978b540356f0a9b4ead384 100644 (file)
--- a/libraries/liblmdb/midl.h
+++ b/libraries/liblmdb/midl.h
@@ -55,7 +55,11 @@ typedef MDB_ID *MDB_IDL;
 /* IDL sizes - likely should be even bigger
  *   limiting factors: sizeof(ID), thread stack size
  */
+#ifdef VL32
+#define        MDB_IDL_LOGN    10      /* DB_SIZE is 2^16, UM_SIZE is 2^17 */
+#else
 #define        MDB_IDL_LOGN    16      /* DB_SIZE is 2^16, UM_SIZE is 2^17 */
+#endif
 #define MDB_IDL_DB_SIZE                (1<<MDB_IDL_LOGN)
 #define MDB_IDL_UM_SIZE                (1<<(MDB_IDL_LOGN+1))
 

diff --git a/libraries/liblmdb/mtest.c b/libraries/liblmdb/mtest.c
index 79b4175e79f740e9c954c9be3c36c429b864037a..2d4e5e8327d515edbabf5636bae21ea36dea1d71 100644 (file)
--- a/libraries/liblmdb/mtest.c
+++ b/libraries/liblmdb/mtest.c
@@ -46,7 +46,7 @@ int main(int argc,char * argv[])
     
                E(mdb_env_create(&env));
                E(mdb_env_set_mapsize(env, 10485760));
-               E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP /*|MDB_NOSYNC*/, 0664));
+               E(mdb_env_open(env, "./testdb", 0 /* MDB_FIXEDMAP |MDB_NOSYNC*/, 0664));
                E(mdb_txn_begin(env, NULL, 0, &txn));
                E(mdb_open(txn, NULL, 0, &dbi));