X-Git-Url: https://git.sur5r.net/?a=blobdiff_plain;f=bacula%2Fpatches%2Ftesting%2Faccurate-db.patch;h=89517d6034a70b75ea4620335afba84da3a01fa1;hb=8fc14a69f0caa1820e4f24b1e7fb4d822759a60d;hp=7f1d5d93216e183933ce81be4dc8d9f7ac6302e3;hpb=9eb4955c3f45f60dcf7859ad0a7846a5f444200f;p=bacula%2Fbacula diff --git a/bacula/patches/testing/accurate-db.patch b/bacula/patches/testing/accurate-db.patch index 7f1d5d9321..89517d6034 100644 --- a/bacula/patches/testing/accurate-db.patch +++ b/bacula/patches/testing/accurate-db.patch @@ -1,303 +1,627 @@ -Index: src/dird/next_vol.c -=================================================================== ---- src/dird/next_vol.c (revision 7286) -+++ src/dird/next_vol.c (working copy) -@@ -94,28 +94,30 @@ - */ - if (prune) { - Dmsg0(150, "Call prune_volumes\n"); -- prune_volumes(jcr, InChanger, mr); -+ ok = prune_volumes(jcr, InChanger, mr); - } -- ok = recycle_oldest_purged_volume(jcr, InChanger, mr); -- if (!ok && create) { -- Dmsg4(050, "after prune volumes_vol ok=%d index=%d InChanger=%d Vstat=%s\n", -- ok, index, InChanger, mr->VolStatus); -- /* -- * 5. Try pulling a volume from the Scratch pool -- */ -- ok = get_scratch_volume(jcr, InChanger, mr); -- } -- /* -- * If we are using an Autochanger and have not found -- * a volume, retry looking for any volume. -- */ -- if (InChanger) { -- InChanger = false; -- if (!ok) { -- continue; /* retry again accepting any volume */ -- } -- } -- } -+ if (!ok) { -+ ok = recycle_oldest_purged_volume(jcr, InChanger, mr); -+ if (!ok && create) { -+ Dmsg4(050, "after prune volumes_vol ok=%d index=%d InChanger=%d Vstat=%s\n", -+ ok, index, InChanger, mr->VolStatus); -+ /* -+ * 5. Try pulling a volume from the Scratch pool -+ */ -+ ok = get_scratch_volume(jcr, InChanger, mr); -+ Dmsg4(050, "after get scratch volume ok=%d index=%d InChanger=%d Vstat=%s\n", -+ ok, index, InChanger, mr->VolStatus); -+ } -+ /* -+ * If we are using an Autochanger and have not found -+ * a volume, retry looking for any volume. -+ */ -+ if (!ok && InChanger) { -+ InChanger = false; -+ continue; /* retry again accepting any volume */ -+ } -+ } -+ } - } - - -Index: src/dird/autoprune.c -=================================================================== ---- src/dird/autoprune.c (revision 7286) -+++ src/dird/autoprune.c (working copy) -@@ -176,6 +176,17 @@ - prune_list.num_ids = 0; /* reset count */ - } - ok = is_volume_purged(ua, &lmr); -+ -+ /* -+ * Check if this volume is available (InChanger + StorageId) -+ * If not, just skip this volume and try the next one -+ */ -+ if (ok && InChanger) { -+ if (!lmr.InChanger || (lmr.StorageId != mr->StorageId)) { -+ ok = false; /* skip this volume, ie not loadable */ -+ } -+ } -+ - /* - * If purged and not moved to another Pool, - * then we stop pruning and take this volume. Index: src/filed/accurate.c =================================================================== ---- src/filed/accurate.c (revision 7286) -+++ src/filed/accurate.c (working copy) -@@ -36,7 +36,7 @@ - static int dbglvl=200; +--- src/filed/accurate.c (révision 7351) ++++ src/filed/accurate.c (copie de travail) +@@ -32,193 +32,311 @@ - typedef struct PrivateCurFile { --#ifndef USE_TCADB -+#ifdef USE_HTABLE - hlink link; - #endif - char *fname; /* not stored with tchdb mode */ -@@ -45,9 +45,9 @@ - bool seen; - } CurFile; + #include "bacula.h" + #include "filed.h" ++#include "lib/htable.h" ++static int dbglvl=500; -+ - #ifdef USE_TCADB +-static int dbglvl=200; ++#ifdef USE_DB ++#include + +-typedef struct PrivateCurFile { +-#ifndef USE_TCADB +- hlink link; +-#endif +- char *fname; /* not stored with tchdb mode */ +- time_t ctime; +- time_t mtime; +- bool seen; +-} CurFile; +- +-#ifdef USE_TCADB -static void realfree(void *p); /* used by tokyo code */ - -+static void realfree(void *p); /* used by tokyo/db code */ /* - * Update hash element seen=1 +- * Update hash element seen=1 ++ * This backend uses DB Berkeley */ -@@ -190,10 +190,150 @@ - return true; - } +-static bool accurate_mark_file_as_seen(JCR *jcr, CurFile *elt) ++class AccurateBackendDB : public AccurateBackend + { +- bool ret=true; ++public: ++ ~AccurateBackendDB() { destroy(); } ++ bool init(JCR *jcr, int nb_elt); ++ void destroy(); --#else /* HTABLE mode */ -+#endif /* USE_TCADB */ -+#ifdef USE_DB +- elt->seen = 1; +- if (!tcadbput(jcr->file_list, +- elt->fname, strlen(elt->fname)+1, +- elt, sizeof(CurFile))) +- { /* TODO: disabling accurate mode ? */ +- Jmsg(jcr, M_ERROR, 1, _("Can't update accurate hash disk\n")); +- ret = false; ++ bool insert(JCR *jcr, char *key, CurFile *item); ++ bool lookup(JCR *jcr, char *key, CurFile *item); ++ ++ bool mark_as_seen(JCR *jcr, char *key, CurFile *item); ++ ++ CurFile *first(CurFile *elt); ++ CurFile *next(); ++ void finish(); ++private: ++ DB *db; /* DB object */ ++ DB_ENV *dbenv; /* DB Env object */ ++ DBC *cursorp; /* DB cursor */ ++ DBT dbkey; ++ DBT dbdata; ++ POOLMEM *hash_name; /* file name for hash */ ++ bool first_pass; ++}; ++ ++void AccurateBackendDB::destroy() ++{ ++ /* cleanup walk cursor if any */ ++ if (cursorp) { ++ cursorp->c_close(cursorp); ++ cursorp = NULL; + } +- +- return ret; ++ if (db) { ++ db->close(db, DB_NOSYNC); ++ db = NULL; ++ } ++ if (dbenv) { ++ dbenv->dbremove(dbenv, NULL, hash_name, NULL, 0); ++ dbenv->close(dbenv, 0); ++ dbenv = NULL; ++ } ++ if (hash_name) { ++ unlink(hash_name); /* remove the hash on disk */ ++ free_pool_memory(hash_name); ++ hash_name = NULL; ++ } + } -+/* -+ * Update hash element seen=1 -+ */ - static bool accurate_mark_file_as_seen(JCR *jcr, CurFile *elt) +-static bool accurate_lookup(JCR *jcr, char *fname, CurFile *ret) ++bool AccurateBackendDB::init(JCR *jcr, int nb_elt) { -+ bool ret=true; -+ DBT dbkey, dbdata; +- bool found=false; +- ret->seen = 0; +- int size; +- CurFile *elt; ++ int ret; ++ ++ Dmsg0(dbglvl, "init DB accurate backend\n"); + +- elt = (CurFile*)tcadbget(jcr->file_list, +- fname, strlen(fname)+1, &size); +- if (elt) ++ if ((ret = db_env_create(&dbenv, 0)) != 0) { ++ Jmsg(jcr, M_ERROR, 1, _("Can't open initialize hash disk ERR=%i\n"), ret); ++ return false; ++ } + -+ elt->seen = 1; ++ dbenv->set_cachesize(dbenv, 0, 256 * 1024 * 1024, 0); ++ dbenv->set_errfile(dbenv, stderr); ++ dbenv->set_errpfx(dbenv, "hash"); ++ ++ int flags = DB_PRIVATE | DB_INIT_MPOOL | DB_CREATE | DB_THREAD; ++ if ((ret = dbenv->open(dbenv, working_directory, flags, 0)) != 0) { ++ destroy(); ++ Jmsg(jcr, M_ERROR, 1, _("Can't open initialize hash disk ERR=%i\n"), ret); ++ return false; ++ } + -+ dbkey.data = elt->fname; -+ dbkey.size = (u_int32_t)strlen(elt->fname)+1; -+ dbdata.data = elt; -+ dbdata.size = sizeof(CurFile); -+ if ((ret = jcr->file_list->put(jcr->file_list, NULL, &dbkey, &dbdata, 0))) { -+ Jmsg(jcr, M_ERROR, 1, _("Can't update accurate hash disk ERR=%i\n"), ret); -+ return 0; ++ dbenv->set_flags(dbenv, DB_TXN_NOSYNC, 1); ++ ++ if ((ret = db_create(&db, dbenv, 0)) != 0) { ++ destroy(); ++ Jmsg(jcr, M_ERROR, 1, _("Can't open accurate hash disk ERR=%i\n"), ret); ++ return false; + } + -+ return 1; ++ hash_name = get_pool_memory(PM_MESSAGE); ++ make_unique_filename(&hash_name, jcr->JobId, "accurate"); ++ ++ if ((ret = db->open(db, ++ NULL, hash_name, NULL, ++ DB_BTREE, DB_CREATE, 0600)) != 0) + { +- /* TODO: don't malloc/free results */ +- found = true; +- elt->fname = fname; +- memcpy(ret, elt, sizeof(CurFile)); +- realfree(elt); +-// Dmsg1(dbglvl, "lookup <%s> ok\n", fname); ++ destroy(); ++ Jmsg(jcr, M_ERROR, 1, _("Can't setup hash disk ERR=%i\n"), ret); ++ return false; + } +- return found; ++ ++ first_pass=0; ++ ++ return true; + } + +-/* Create tokyo dbm hash file +- * If something goes wrong, we cancel accurate mode. +- */ +-static bool accurate_init(JCR *jcr, int nbfile) ++/* Just update the element->seen to know if we have seen it */ ++bool AccurateBackendDB::mark_as_seen(JCR *jcr, char *key, CurFile *item) + { +- jcr->file_list = tcadbnew(); +-// +-// tchdbsetcache(jcr->file_list, 300000); +-// tchdbtune(jcr->file_list, +-// nbfile, /* nb bucket 0.5n to 4n */ +-// 6, /* size of element 2^x */ +-// 16, +-// 0); /* options like compression */ +-// +- jcr->hash_name = get_pool_memory(PM_MESSAGE); +- POOLMEM *temp = get_pool_memory(PM_MESSAGE); ++ item->seen = 1; ++ return insert(jcr, key, item); ++} + +- if (nbfile > 500000) { +- make_unique_filename(&jcr->hash_name, jcr->JobId, "accurate"); +- pm_strcat(jcr->hash_name, ".tcb"); +- Mmsg(temp, "%s#bnum=%i#mode=e#opts=l", +- jcr->hash_name, nbfile*4); +- Dmsg1(dbglvl, "Doing accurate hash on disk %s\n", jcr->hash_name); +- } else { +- Dmsg0(dbglvl, "Doing accurate hash on memory\n"); +- pm_strcpy(jcr->hash_name, "*"); +- pm_strcpy(temp, "*"); ++/* insert/replace */ ++bool AccurateBackendDB::insert(JCR *jcr, char *key, CurFile *item) ++{ ++ int ret; ++ memset(&dbkey, 0, sizeof(DBT)); ++ memset(&dbdata, 0, sizeof(DBT)); ++ dbkey.data = key; ++ dbkey.size = strlen(key)+1; ++ dbdata.data = item; ++ dbdata.size = sizeof(CurFile); ++ if ((ret = db->put(db, NULL, &dbkey, &dbdata, 0))) { ++ Jmsg(jcr, M_ERROR, 1, _("Can't update accurate hash disk ERR=%i\n"), ret); ++ return false; + } ++ return true; +} + -+static bool accurate_lookup(JCR *jcr, char *fname, CurFile *ret) ++bool AccurateBackendDB::lookup(JCR *jcr, char *key, CurFile *item) +{ -+ ret->seen = 0; -+ DBT dbkey, dbdata; ++ int ret=false; ++ + /* Zero out the DBTs before using them. */ + memset(&dbkey, 0, sizeof(DBT)); + memset(&dbdata, 0, sizeof(DBT)); + -+ dbkey.data = fname; -+ dbkey.ulen = strlen(fname)+1; -+ -+ dbdata.data = ret; ++ dbkey.data = key; ++ dbkey.size = strlen(key)+1; + +- if(!tcadbopen(jcr->file_list, jcr->hash_name)){ +- Jmsg(jcr, M_ERROR, 1, _("Can't open accurate hash disk\n")); +- Jmsg(jcr, M_INFO, 1, _("Disabling accurate mode\n")); +- tcadbdel(jcr->file_list); +- jcr->file_list = NULL; +- jcr->accurate = false; ++ dbdata.data = item; + dbdata.ulen = sizeof(CurFile); + dbdata.flags = DB_DBT_USERMEM; + -+ if (jcr->file_list->get(jcr->file_list, NULL, &dbkey, &dbdata, 0) == 0) { -+ return 1; -+ } else { -+ return 0; -+ } ++ if (db->get(db, NULL, &dbkey, &dbdata, 0) == 0) { ++ ret=true; + } +- free_pool_memory(temp); +- return jcr->file_list != NULL; ++ item->fname = key; ++ return ret; + } + +-/* This function is called at the end of backup +- * We walk over all hash disk element, and we check +- * for elt.seen. ++/* ++ * We use user memory to copy data + */ +-bool accurate_send_deleted_list(JCR *jcr) ++CurFile *AccurateBackendDB::first(CurFile *elt) + { +- char *key; ++ /* Zero out the DBTs before using them. */ ++ memset(&dbkey, 0, sizeof(DBT)); ++ memset(&dbdata, 0, sizeof(DBT)); ++ dbdata.data = elt; ++ dbdata.ulen = sizeof(CurFile); ++ dbdata.flags = DB_DBT_USERMEM; ++ db->cursor(db, NULL, &cursorp, 0); ++ return next(); +} + -+/* Create tokyo dbm hash file -+ * If something goes wrong, we cancel accurate mode. ++CurFile *AccurateBackendDB::next() ++{ + CurFile *elt; +- int size; +- FF_PKT *ff_pkt; +- int stream = STREAM_UNIX_ATTRIBUTES; +- +- if (!jcr->accurate || jcr->JobLevel == L_FULL) { +- goto bail_out; ++ if (cursorp->c_get(cursorp, &dbkey, &dbdata, DB_NEXT) == 0) { ++ /* update fname field with fresh data */ ++ elt = (CurFile *)dbdata.data; ++ elt->fname = (char *)dbkey.data; ++ return elt; ++ } else { ++ return NULL; + } ++} + +- if (jcr->file_list == NULL) { +- goto bail_out; ++void AccurateBackendDB::finish() ++{ ++ if (cursorp) { ++ cursorp->c_close(cursorp); ++ cursorp = NULL; + } ++} + +- ff_pkt = init_find_files(); +- ff_pkt->type = FT_DELETED; ++#endif /* USE_DB */ + +- /* traverse records */ +- tcadbiterinit(jcr->file_list); +- while((key = tcadbiternext2(jcr->file_list)) != NULL) { +- elt = (CurFile *) tcadbget(jcr->file_list, +- key, strlen(key)+1, &size); +- if (elt) +- { +- if (!elt->seen) { /* already seen */ +- ff_pkt->fname = key; +- ff_pkt->statp.st_mtime = elt->mtime; +- ff_pkt->statp.st_ctime = elt->ctime; +- encode_and_send_attributes(jcr, ff_pkt, stream); +- } +- realfree(elt); +- } +- realfree(key); /* tokyo cabinet have to use real free() */ +- } ++/****************************************************************/ + +- term_find_files(ff_pkt); +-bail_out: +- /* TODO: clean htable when this function is not reached ? */ +- if (jcr->file_list) { +- if(!tcadbclose(jcr->file_list)){ +- Jmsg(jcr, M_ERROR, 1, _("Can't close accurate hash disk\n")); +- } ++/* ++ * This is the htable implementation for accurate mode + */ -+static bool accurate_init(JCR *jcr, int nbfile) ++class AccurateBackendHT : public AccurateBackend +{ -+ int ret; -+ -+ if ((ret = db_create(&jcr->file_list, NULL, 0)) != 0) { -+ Jmsg(jcr, M_ERROR, 1, _("Can't open accurate hash disk\n")); -+ return 0; -+ } -+ -+ jcr->file_list->set_errfile(jcr->file_list, stderr); -+ jcr->file_list->set_errpfx(jcr->file_list, "hash"); -+ -+ if ((ret = jcr->file_list->set_cachesize(jcr->file_list, 0, 32 * 1024 * 1024, 0)) != 0) { -+ Jmsg(jcr, M_ERROR, 1, _("Can't setup hash disk cache size\n")); -+ } ++public: ++ ~AccurateBackendHT() { destroy(); } ++ bool init(JCR *jcr, int nb_elt); ++ void destroy(); + +- /* delete the object */ +- tcadbdel(jcr->file_list); +- if (!bstrcmp(jcr->hash_name, "*")) { +- unlink(jcr->hash_name); +- } ++ bool insert(JCR *jcr, char *key, CurFile *item); ++ bool lookup(JCR *jcr, char *key, CurFile *item); + +- free_pool_memory(jcr->hash_name); +- jcr->hash_name = NULL; +- jcr->file_list = NULL; ++ bool mark_as_seen(JCR *jcr, char *key, CurFile *item); + -+ jcr->hash_name = get_pool_memory(PM_MESSAGE); -+ make_unique_filename(&jcr->hash_name, jcr->JobId, "accurate"); ++ CurFile *first(CurFile *elt); ++ CurFile *next(); ++ void finish(); ++private: ++ htable *db; ++}; + -+ if ((ret = jcr->file_list->open(jcr->file_list, -+ NULL, jcr->hash_name, NULL, DB_BTREE, DB_CREATE, 0600)) != 0) { -+ jcr->file_list->close(jcr->file_list, DB_NOSYNC); -+ jcr->file_list=NULL; -+ free_pool_memory(jcr->hash_name); -+ jcr->hash_name = NULL; -+ Jmsg(jcr, M_ERROR, 1, _("Can't setup hash disk cache size\n")); -+ return 0; -+ } ++typedef struct { ++ CurFile elt; ++ hlink link; /* need this for htable link */ ++} HTCurFile; + -+ return jcr->file_list != NULL; ++void AccurateBackendHT::destroy() ++{ ++ if (db) { ++ db->destroy(); ++ free(db); ++ db = NULL; + } +} + -+/* This function is called at the end of backup -+ * We walk over all hash disk element, and we check -+ * for elt.seen. -+ */ -+bool accurate_send_deleted_list(JCR *jcr) ++bool AccurateBackendHT::init(JCR *jcr, int nb_elt) +{ -+ CurFile elt; -+ FF_PKT *ff_pkt; -+ int stream = STREAM_UNIX_ATTRIBUTES; -+ DBC *cursorp=NULL; -+ DBT dbkey, dbdata; -+ /* Zero out the DBTs before using them. */ -+ memset(&dbkey, 0, sizeof(DBT)); -+ memset(&dbdata, 0, sizeof(DBT)); -+ dbdata.data = &elt; -+ dbdata.ulen = sizeof(CurFile); -+ dbdata.flags = DB_DBT_USERMEM; -+ -+ if (!jcr->accurate || jcr->JobLevel == L_FULL) { -+ goto bail_out; -+ } ++ Dmsg0(dbglvl, "init HT accurate backend\n"); + -+ if (jcr->file_list == NULL) { -+ goto bail_out; -+ } -+ -+ ff_pkt = init_find_files(); -+ ff_pkt->type = FT_DELETED; ++ HTCurFile *elt=NULL; ++ db = (htable *)malloc(sizeof(htable)); ++ db->init(elt, &elt->link, nb_elt); + return true; + } + +-#else /* HTABLE mode */ ++bool AccurateBackendHT::insert(JCR *jcr, char *key, CurFile *item) ++{ ++ /* alloc CurFile + hlink + fname */ ++ HTCurFile *htf = (HTCurFile *)db->hash_malloc(sizeof(HTCurFile)+strlen(key)+1); ++ memcpy(&htf->elt, item, sizeof(CurFile)); + +-static bool accurate_mark_file_as_seen(JCR *jcr, CurFile *elt) ++ /* store fname at the end of the struct */ ++ htf->elt.fname = (char *) htf + sizeof(HTCurFile); + -+ /* traverse records */ -+ jcr->file_list->cursor(jcr->file_list, NULL, &cursorp, 0); ++ strcpy(htf->elt.fname, key); ++ db->insert(htf->elt.fname, htf); ++ return true; ++} + -+ while(cursorp->c_get(cursorp, &dbkey, &dbdata, DB_NEXT) == 0) { -+ if (!elt.seen) { /* already seen */ -+ ff_pkt->fname = (char *) dbkey.data; -+ ff_pkt->statp.st_mtime = elt.mtime; -+ ff_pkt->statp.st_ctime = elt.ctime; -+ encode_and_send_attributes(jcr, ff_pkt, stream); -+ } ++bool AccurateBackendHT::mark_as_seen(JCR *jcr, char *key, CurFile *item) + { +- CurFile *temp = (CurFile *)jcr->file_list->lookup(elt->fname); +- temp->seen = 1; /* records are in memory */ ++ HTCurFile *temp = (HTCurFile *)db->lookup(key); ++ if (temp) { ++ temp->elt.seen = 1; /* update in memory */ + } -+ cursorp->c_close(cursorp); + return true; + } + +-static bool accurate_lookup(JCR *jcr, char *fname, CurFile *ret) ++bool AccurateBackendHT::lookup(JCR *jcr, char *key, CurFile *item) + { + bool found=false; +- ret->seen = 0; + +- CurFile *temp = (CurFile *)jcr->file_list->lookup(fname); ++ HTCurFile *temp = (HTCurFile *)db->lookup(key); + if (temp) { +- memcpy(ret, temp, sizeof(CurFile)); ++ memcpy(item, &temp->elt, sizeof(CurFile)); + found=true; +-// Dmsg1(dbglvl, "lookup <%s> ok\n", fname); + } +- + return found; + } + ++CurFile *AccurateBackendHT::first(CurFile *elt) ++{ ++ HTCurFile *temp = (HTCurFile *)db->first(); ++ return &temp->elt; ++} + -+ term_find_files(ff_pkt); -+bail_out: ++CurFile *AccurateBackendHT::next() ++{ ++ HTCurFile *temp = (HTCurFile *)db->next(); ++ return &temp->elt; ++} + -+ if (jcr->file_list) { -+ jcr->file_list->close(jcr->file_list, DB_NOSYNC); -+ if (!bstrcmp(jcr->hash_name, "*")) { -+ unlink(jcr->hash_name); -+ } -+ free_pool_memory(jcr->hash_name); -+ jcr->hash_name = NULL; -+ jcr->file_list = NULL; -+ } -+ return true; ++void AccurateBackendHT::finish() ++{ +} + -+#endif /* USE_DB */ -+#ifdef USE_HTABLE ++/****************************************************************/ ++ ++/* Create hash file ++ * For less than 1M files, use htable in memory ++ */ + static bool accurate_init(JCR *jcr, int nbfile) + { +- CurFile *elt=NULL; +- jcr->file_list = (htable *)malloc(sizeof(htable)); +- jcr->file_list->init(elt, &elt->link, nbfile); ++#ifdef USE_DB ++ if (nbfile > 1000000) { ++ jcr->file_list = New(AccurateBackendDB); ++ } else { ++ jcr->file_list = New(AccurateBackendHT); ++ } ++#else ++ jcr->file_list = New(AccurateBackendHT); ++#endif ++ jcr->file_list->init(jcr, nbfile); + -+static bool accurate_mark_file_as_seen(JCR *jcr, CurFile *elt) -+{ - CurFile *temp = (CurFile *)jcr->file_list->lookup(elt->fname); - temp->seen = 1; /* records are in memory */ return true; -@@ -286,7 +426,22 @@ - Jmsg(jcr, M_ERROR, 1, _("Can't update accurate hash disk ERR=%s\n")); + } + +@@ -228,7 +346,8 @@ + */ + bool accurate_send_deleted_list(JCR *jcr) + { +- CurFile *elt; ++ CurFile elt; ++ CurFile *item; + FF_PKT *ff_pkt; + int stream = STREAM_UNIX_ATTRIBUTES; + +@@ -242,31 +361,28 @@ + + ff_pkt = init_find_files(); + ff_pkt->type = FT_DELETED; +- +- foreach_htable (elt, jcr->file_list) { +- if (!elt->seen) { /* already seen */ +- Dmsg2(dbglvl, "deleted fname=%s seen=%i\n", elt->fname, elt->seen); +- ff_pkt->fname = elt->fname; +- ff_pkt->statp.st_mtime = elt->mtime; +- ff_pkt->statp.st_ctime = elt->ctime; ++ ++ for (item = jcr->file_list->first(&elt); item ; item = jcr->file_list->next()) { ++ if (!item->seen) { /* already seen */ ++ Dmsg2(dbglvl, "deleted fname=%s seen=%i\n", item->fname, item->seen); ++ ff_pkt->fname = item->fname; ++ ff_pkt->statp.st_mtime = item->mtime; ++ ff_pkt->statp.st_ctime = item->ctime; + encode_and_send_attributes(jcr, ff_pkt, stream); + } +-// free(elt->fname); + } ++ jcr->file_list->finish(); + + term_find_files(ff_pkt); + bail_out: + /* TODO: clean htable when this function is not reached ? */ + if (jcr->file_list) { +- jcr->file_list->destroy(); +- free(jcr->file_list); ++ delete jcr->file_list; + jcr->file_list = NULL; + } + return true; + } + +-#endif /* common code */ +- + static bool accurate_add_file(JCR *jcr, char *fname, char *lstat) + { + bool ret = true; +@@ -278,25 +394,12 @@ + elt.mtime = statp.st_mtime; + elt.seen = 0; + +-#ifdef USE_TCADB +- if (!tcadbput(jcr->file_list, +- fname, strlen(fname)+1, +- &elt, sizeof(CurFile))) +- { +- Jmsg(jcr, M_ERROR, 1, _("Can't update accurate hash disk ERR=%s\n")); ++ if (!jcr->file_list->insert(jcr, fname, &elt)) { ++ Jmsg(jcr, M_ERROR, 1, _("Can't update accurate hash ERR=%s\n")); ret = false; } -#else /* HTABLE */ -+#endif -+#ifdef USE_DB -+ DBT dbkey, dbdata; -+ memset(&dbkey, 0, sizeof(DBT)); -+ memset(&dbdata, 0, sizeof(DBT)); -+ dbkey.data = elt.fname; -+ dbkey.size = (u_int32_t)strlen(elt.fname)+1; -+ dbdata.data = &elt; -+ dbdata.size = sizeof(CurFile); -+ if ((ret = jcr->file_list->put(jcr->file_list, NULL, &dbkey, &dbdata, 0))) { -+ Jmsg(jcr, M_ERROR, 1, _("Can't update accurate hash disk ERR=%i\n"), ret); -+ return false; -+ } -+ -+#endif -+#ifdef USE_HTABLE - CurFile *item; - /* we store CurFile, fname and ctime/mtime in the same chunk */ - item = (CurFile *)jcr->file_list->hash_malloc(sizeof(CurFile)+strlen(fname)+1); +- CurFile *item; +- /* we store CurFile, fname and ctime/mtime in the same chunk */ +- item = (CurFile *)jcr->file_list->hash_malloc(sizeof(CurFile)+strlen(fname)+1); +- memcpy(item, &elt, sizeof(CurFile)); +- item->fname = (char *)item+sizeof(CurFile); +- strcpy(item->fname, fname); +- jcr->file_list->insert(item->fname, item); +-#endif + +-// Dmsg2(dbglvl, "add fname=<%s> lstat=%s\n", fname, lstat); ++ Dmsg2(dbglvl, "add fname=<%s> lstat=%s\n", fname, lstat); + return ret; + } + +@@ -324,7 +427,7 @@ + fname = ff_pkt->fname; + } + +- if (!accurate_lookup(jcr, fname, &elt)) { ++ if (!jcr->file_list->lookup(jcr, fname, &elt)) { + Dmsg1(dbglvl, "accurate %s (not found)\n", fname); + stat = true; + goto bail_out; +@@ -343,7 +446,7 @@ + stat = true; + } + +- accurate_mark_file_as_seen(jcr, &elt); ++ jcr->file_list->mark_as_seen(jcr, fname, &elt); + Dmsg2(dbglvl, "accurate %s = %i\n", fname, stat); + + bail_out: +@@ -379,7 +482,7 @@ + while (dir->recv() >= 0) { + len = strlen(dir->msg) + 1; + if (len < dir->msglen) { +- accurate_add_file(jcr, dir->msg, dir->msg + len); ++ accurate_add_file(jcr, dir->msg, dir->msg + len); + } + } + +@@ -398,17 +501,3 @@ + + return true; + } +- +-#ifdef USE_TCADB +- +-/* +- * Tokyo Cabinet library doesn't use smartalloc by default +- * results need to be released with real free() +- */ +-#undef free +-void realfree(void *p) +-{ +- free(p); +-} +- +-#endif Index: src/filed/filed.h =================================================================== ---- src/filed/filed.h (revision 7286) -+++ src/filed/filed.h (working copy) -@@ -37,7 +37,11 @@ +--- src/filed/filed.h (révision 7351) ++++ src/filed/filed.h (copie de travail) +@@ -35,11 +35,35 @@ + + #define FILE_DAEMON 1 - #ifdef USE_TCADB /* hash disk based */ - # include +-#ifdef USE_TCADB /* hash disk based */ +-# include -#else -+#endif -+#ifdef USE_DB -+# include -+#endif -+#ifdef USE_HTABLE - # include "lib/htable.h" - #endif +-# include "lib/htable.h" +-#endif ++ ++/* ++ * Used to store accurate information ++ */ ++typedef struct { ++ char *fname; ++ time_t ctime; ++ time_t mtime; ++ bool seen; ++} CurFile; ++ ++/* ++ * Virtual class for accurate backend (libdb or htable) ++ */ ++class AccurateBackend: public SMARTALLOC ++{ ++public: ++ AccurateBackend() : SMARTALLOC() {} ++ virtual ~AccurateBackend() {} ++ virtual bool init(JCR *jcr, int nb_elt) = 0; ++ virtual bool insert(JCR *jcr, char *key, CurFile *item) = 0; ++ virtual bool lookup(JCR *jcr, char *key, CurFile *item) = 0; ++ virtual bool mark_as_seen(JCR *jcr, char *key, CurFile *item) = 0; ++ virtual CurFile *first(CurFile *elt) = 0; ++ virtual CurFile *next() = 0; ++ virtual void finish() = 0; ++ virtual void destroy() {} ++}; ++ #include "filed_conf.h" + #include "fd_plugins.h" + #include "findlib/find.h" Index: src/baconfig.h =================================================================== ---- src/baconfig.h (revision 7286) -+++ src/baconfig.h (working copy) +--- src/baconfig.h (révision 7351) ++++ src/baconfig.h (copie de travail) @@ -113,6 +113,12 @@ #endif /* HAVE_WIN32 */ @@ -311,34 +635,21 @@ Index: src/baconfig.h #ifdef ENABLE_NLS #include -Index: src/win32/build-depkgs-mingw32 -=================================================================== ---- src/win32/build-depkgs-mingw32 (revision 7286) -+++ src/win32/build-depkgs-mingw32 (working copy) -@@ -519,7 +519,7 @@ - process_cmd_utils - process_mkisofs - process_dvd_rw_tools -- process_qt4 -+# process_qt4 - else - for dependency in "$@" - do Index: src/jcr.h =================================================================== ---- src/jcr.h (revision 7286) -+++ src/jcr.h (working copy) -@@ -344,7 +344,12 @@ - #ifdef USE_TCADB - TCADB *file_list; /* Previous file list (accurate mode) */ - POOLMEM *hash_name; +--- src/jcr.h (révision 7351) ++++ src/jcr.h (copie de travail) +@@ -341,12 +341,7 @@ + CRYPTO_CTX crypto; /* Crypto ctx */ + DIRRES* director; /* Director resource */ + bool VSS; /* VSS used by FD */ +-#ifdef USE_TCADB +- TCADB *file_list; /* Previous file list (accurate mode) */ +- POOLMEM *hash_name; -#else -+#endif -+#ifdef USE_DB -+ DB *file_list; -+ POOLMEM *hash_name; -+#endif -+#ifdef USE_HTABLE - htable *file_list; /* Previous file list (accurate mode) */ - #endif +- htable *file_list; /* Previous file list (accurate mode) */ +-#endif ++ AccurateBackend *file_list; /* Accurate backend store */ #endif /* FILE_DAEMON */ + +