1 Index: src/filed/accurate.c
2 ===================================================================
3 --- src/filed/accurate.c (révision 7351)
4 +++ src/filed/accurate.c (copie de travail)
9 +#include "lib/htable.h"
10 +static int dbglvl=500;
12 -static int dbglvl=200;
16 -typedef struct PrivateCurFile {
20 - char *fname; /* not stored with tchdb mode */
27 -static void realfree(void *p); /* used by tokyo code */
30 - * Update hash element seen=1
31 + * This backend uses DB Berkeley
33 -static bool accurate_mark_file_as_seen(JCR *jcr, CurFile *elt)
34 +class AccurateBackendDB : public AccurateBackend
38 + ~AccurateBackendDB() { destroy(); }
39 + bool init(JCR *jcr, int nb_elt);
43 - if (!tcadbput(jcr->file_list,
44 - elt->fname, strlen(elt->fname)+1,
45 - elt, sizeof(CurFile)))
46 - { /* TODO: disabling accurate mode ? */
47 - Jmsg(jcr, M_ERROR, 1, _("Can't update accurate hash disk\n"));
49 + bool insert(JCR *jcr, char *key, CurFile *item);
50 + bool lookup(JCR *jcr, char *key, CurFile *item);
52 + bool mark_as_seen(JCR *jcr, char *key, CurFile *item);
54 + CurFile *first(CurFile *elt);
58 + DB *db; /* DB object */
59 + DB_ENV *dbenv; /* DB Env object */
60 + DBC *cursorp; /* DB cursor */
63 + POOLMEM *hash_name; /* file name for hash */
67 +void AccurateBackendDB::destroy()
69 + /* cleanup walk cursor if any */
71 + cursorp->c_close(cursorp);
77 + db->close(db, DB_NOSYNC);
81 + dbenv->dbremove(dbenv, NULL, hash_name, NULL, 0);
82 + dbenv->close(dbenv, 0);
86 + unlink(hash_name); /* remove the hash on disk */
87 + free_pool_memory(hash_name);
92 -static bool accurate_lookup(JCR *jcr, char *fname, CurFile *ret)
93 +bool AccurateBackendDB::init(JCR *jcr, int nb_elt)
101 + Dmsg0(dbglvl, "init DB accurate backend\n");
103 - elt = (CurFile*)tcadbget(jcr->file_list,
104 - fname, strlen(fname)+1, &size);
106 + if ((ret = db_env_create(&dbenv, 0)) != 0) {
107 + Jmsg(jcr, M_ERROR, 1, _("Can't open initialize hash disk ERR=%i\n"), ret);
111 + dbenv->set_cachesize(dbenv, 0, 256 * 1024 * 1024, 0);
112 + dbenv->set_errfile(dbenv, stderr);
113 + dbenv->set_errpfx(dbenv, "hash");
115 + int flags = DB_PRIVATE | DB_INIT_MPOOL | DB_CREATE | DB_THREAD;
116 + if ((ret = dbenv->open(dbenv, working_directory, flags, 0)) != 0) {
118 + Jmsg(jcr, M_ERROR, 1, _("Can't open initialize hash disk ERR=%i\n"), ret);
122 + dbenv->set_flags(dbenv, DB_TXN_NOSYNC, 1);
124 + if ((ret = db_create(&db, dbenv, 0)) != 0) {
126 + Jmsg(jcr, M_ERROR, 1, _("Can't open accurate hash disk ERR=%i\n"), ret);
130 + hash_name = get_pool_memory(PM_MESSAGE);
131 + make_unique_filename(&hash_name, jcr->JobId, "accurate");
133 + if ((ret = db->open(db,
134 + NULL, hash_name, NULL,
135 + DB_BTREE, DB_CREATE, 0600)) != 0)
137 - /* TODO: don't malloc/free results */
139 - elt->fname = fname;
140 - memcpy(ret, elt, sizeof(CurFile));
142 -// Dmsg1(dbglvl, "lookup <%s> ok\n", fname);
144 + Jmsg(jcr, M_ERROR, 1, _("Can't setup hash disk ERR=%i\n"), ret);
154 -/* Create tokyo dbm hash file
155 - * If something goes wrong, we cancel accurate mode.
157 -static bool accurate_init(JCR *jcr, int nbfile)
158 +/* Just update the element->seen to know if we have seen it */
159 +bool AccurateBackendDB::mark_as_seen(JCR *jcr, char *key, CurFile *item)
161 - jcr->file_list = tcadbnew();
163 -// tchdbsetcache(jcr->file_list, 300000);
164 -// tchdbtune(jcr->file_list,
165 -// nbfile, /* nb bucket 0.5n to 4n */
166 -// 6, /* size of element 2^x */
168 -// 0); /* options like compression */
170 - jcr->hash_name = get_pool_memory(PM_MESSAGE);
171 - POOLMEM *temp = get_pool_memory(PM_MESSAGE);
173 + return insert(jcr, key, item);
176 - if (nbfile > 500000) {
177 - make_unique_filename(&jcr->hash_name, jcr->JobId, "accurate");
178 - pm_strcat(jcr->hash_name, ".tcb");
179 - Mmsg(temp, "%s#bnum=%i#mode=e#opts=l",
180 - jcr->hash_name, nbfile*4);
181 - Dmsg1(dbglvl, "Doing accurate hash on disk %s\n", jcr->hash_name);
183 - Dmsg0(dbglvl, "Doing accurate hash on memory\n");
184 - pm_strcpy(jcr->hash_name, "*");
185 - pm_strcpy(temp, "*");
186 +/* insert/replace */
187 +bool AccurateBackendDB::insert(JCR *jcr, char *key, CurFile *item)
190 + memset(&dbkey, 0, sizeof(DBT));
191 + memset(&dbdata, 0, sizeof(DBT));
193 + dbkey.size = strlen(key)+1;
194 + dbdata.data = item;
195 + dbdata.size = sizeof(CurFile);
196 + if ((ret = db->put(db, NULL, &dbkey, &dbdata, 0))) {
197 + Jmsg(jcr, M_ERROR, 1, _("Can't update accurate hash disk ERR=%i\n"), ret);
203 +bool AccurateBackendDB::lookup(JCR *jcr, char *key, CurFile *item)
207 + /* Zero out the DBTs before using them. */
208 + memset(&dbkey, 0, sizeof(DBT));
209 + memset(&dbdata, 0, sizeof(DBT));
212 + dbkey.size = strlen(key)+1;
214 - if(!tcadbopen(jcr->file_list, jcr->hash_name)){
215 - Jmsg(jcr, M_ERROR, 1, _("Can't open accurate hash disk\n"));
216 - Jmsg(jcr, M_INFO, 1, _("Disabling accurate mode\n"));
217 - tcadbdel(jcr->file_list);
218 - jcr->file_list = NULL;
219 - jcr->accurate = false;
220 + dbdata.data = item;
221 + dbdata.ulen = sizeof(CurFile);
222 + dbdata.flags = DB_DBT_USERMEM;
224 + if (db->get(db, NULL, &dbkey, &dbdata, 0) == 0) {
227 - free_pool_memory(temp);
228 - return jcr->file_list != NULL;
233 -/* This function is called at the end of backup
234 - * We walk over all hash disk element, and we check
237 + * We use user memory to copy data
239 -bool accurate_send_deleted_list(JCR *jcr)
240 +CurFile *AccurateBackendDB::first(CurFile *elt)
243 + /* Zero out the DBTs before using them. */
244 + memset(&dbkey, 0, sizeof(DBT));
245 + memset(&dbdata, 0, sizeof(DBT));
247 + dbdata.ulen = sizeof(CurFile);
248 + dbdata.flags = DB_DBT_USERMEM;
249 + db->cursor(db, NULL, &cursorp, 0);
253 +CurFile *AccurateBackendDB::next()
258 - int stream = STREAM_UNIX_ATTRIBUTES;
260 - if (!jcr->accurate || jcr->JobLevel == L_FULL) {
262 + if (cursorp->c_get(cursorp, &dbkey, &dbdata, DB_NEXT) == 0) {
263 + /* update fname field with fresh data */
264 + elt = (CurFile *)dbdata.data;
265 + elt->fname = (char *)dbkey.data;
272 - if (jcr->file_list == NULL) {
274 +void AccurateBackendDB::finish()
277 + cursorp->c_close(cursorp);
282 - ff_pkt = init_find_files();
283 - ff_pkt->type = FT_DELETED;
286 - /* traverse records */
287 - tcadbiterinit(jcr->file_list);
288 - while((key = tcadbiternext2(jcr->file_list)) != NULL) {
289 - elt = (CurFile *) tcadbget(jcr->file_list,
290 - key, strlen(key)+1, &size);
293 - if (!elt->seen) { /* already seen */
294 - ff_pkt->fname = key;
295 - ff_pkt->statp.st_mtime = elt->mtime;
296 - ff_pkt->statp.st_ctime = elt->ctime;
297 - encode_and_send_attributes(jcr, ff_pkt, stream);
301 - realfree(key); /* tokyo cabinet have to use real free() */
303 +/****************************************************************/
305 - term_find_files(ff_pkt);
307 - /* TODO: clean htable when this function is not reached ? */
308 - if (jcr->file_list) {
309 - if(!tcadbclose(jcr->file_list)){
310 - Jmsg(jcr, M_ERROR, 1, _("Can't close accurate hash disk\n"));
313 + * This is the htable implementation for accurate mode
315 +class AccurateBackendHT : public AccurateBackend
318 + ~AccurateBackendHT() { destroy(); }
319 + bool init(JCR *jcr, int nb_elt);
322 - /* delete the object */
323 - tcadbdel(jcr->file_list);
324 - if (!bstrcmp(jcr->hash_name, "*")) {
325 - unlink(jcr->hash_name);
327 + bool insert(JCR *jcr, char *key, CurFile *item);
328 + bool lookup(JCR *jcr, char *key, CurFile *item);
330 - free_pool_memory(jcr->hash_name);
331 - jcr->hash_name = NULL;
332 - jcr->file_list = NULL;
333 + bool mark_as_seen(JCR *jcr, char *key, CurFile *item);
335 + CurFile *first(CurFile *elt);
344 + hlink link; /* need this for htable link */
347 +void AccurateBackendHT::destroy()
356 +bool AccurateBackendHT::init(JCR *jcr, int nb_elt)
358 + Dmsg0(dbglvl, "init HT accurate backend\n");
360 + HTCurFile *elt=NULL;
361 + db = (htable *)malloc(sizeof(htable));
362 + db->init(elt, &elt->link, nb_elt);
366 -#else /* HTABLE mode */
367 +bool AccurateBackendHT::insert(JCR *jcr, char *key, CurFile *item)
369 + /* alloc CurFile + hlink + fname */
370 + HTCurFile *htf = (HTCurFile *)db->hash_malloc(sizeof(HTCurFile)+strlen(key)+1);
371 + memcpy(&htf->elt, item, sizeof(CurFile));
373 -static bool accurate_mark_file_as_seen(JCR *jcr, CurFile *elt)
374 + /* store fname at the end of the struct */
375 + htf->elt.fname = (char *) htf + sizeof(HTCurFile);
377 + strcpy(htf->elt.fname, key);
378 + db->insert(htf->elt.fname, htf);
382 +bool AccurateBackendHT::mark_as_seen(JCR *jcr, char *key, CurFile *item)
384 - CurFile *temp = (CurFile *)jcr->file_list->lookup(elt->fname);
385 - temp->seen = 1; /* records are in memory */
386 + HTCurFile *temp = (HTCurFile *)db->lookup(key);
388 + temp->elt.seen = 1; /* update in memory */
393 -static bool accurate_lookup(JCR *jcr, char *fname, CurFile *ret)
394 +bool AccurateBackendHT::lookup(JCR *jcr, char *key, CurFile *item)
399 - CurFile *temp = (CurFile *)jcr->file_list->lookup(fname);
400 + HTCurFile *temp = (HTCurFile *)db->lookup(key);
402 - memcpy(ret, temp, sizeof(CurFile));
403 + memcpy(item, &temp->elt, sizeof(CurFile));
405 -// Dmsg1(dbglvl, "lookup <%s> ok\n", fname);
411 +CurFile *AccurateBackendHT::first(CurFile *elt)
413 + HTCurFile *temp = (HTCurFile *)db->first();
417 +CurFile *AccurateBackendHT::next()
419 + HTCurFile *temp = (HTCurFile *)db->next();
423 +void AccurateBackendHT::finish()
427 +/****************************************************************/
430 + * For less than 1M files, use htable in memory
432 static bool accurate_init(JCR *jcr, int nbfile)
435 - jcr->file_list = (htable *)malloc(sizeof(htable));
436 - jcr->file_list->init(elt, &elt->link, nbfile);
438 + if (nbfile > 1000000) {
439 + jcr->file_list = New(AccurateBackendDB);
441 + jcr->file_list = New(AccurateBackendHT);
444 + jcr->file_list = New(AccurateBackendHT);
446 + jcr->file_list->init(jcr, nbfile);
453 bool accurate_send_deleted_list(JCR *jcr)
459 int stream = STREAM_UNIX_ATTRIBUTES;
461 @@ -242,31 +361,28 @@
463 ff_pkt = init_find_files();
464 ff_pkt->type = FT_DELETED;
466 - foreach_htable (elt, jcr->file_list) {
467 - if (!elt->seen) { /* already seen */
468 - Dmsg2(dbglvl, "deleted fname=%s seen=%i\n", elt->fname, elt->seen);
469 - ff_pkt->fname = elt->fname;
470 - ff_pkt->statp.st_mtime = elt->mtime;
471 - ff_pkt->statp.st_ctime = elt->ctime;
473 + for (item = jcr->file_list->first(&elt); item ; item = jcr->file_list->next()) {
474 + if (!item->seen) { /* already seen */
475 + Dmsg2(dbglvl, "deleted fname=%s seen=%i\n", item->fname, item->seen);
476 + ff_pkt->fname = item->fname;
477 + ff_pkt->statp.st_mtime = item->mtime;
478 + ff_pkt->statp.st_ctime = item->ctime;
479 encode_and_send_attributes(jcr, ff_pkt, stream);
481 -// free(elt->fname);
483 + jcr->file_list->finish();
485 term_find_files(ff_pkt);
487 /* TODO: clean htable when this function is not reached ? */
488 if (jcr->file_list) {
489 - jcr->file_list->destroy();
490 - free(jcr->file_list);
491 + delete jcr->file_list;
492 jcr->file_list = NULL;
497 -#endif /* common code */
499 static bool accurate_add_file(JCR *jcr, char *fname, char *lstat)
502 @@ -278,25 +394,12 @@
503 elt.mtime = statp.st_mtime;
507 - if (!tcadbput(jcr->file_list,
508 - fname, strlen(fname)+1,
509 - &elt, sizeof(CurFile)))
511 - Jmsg(jcr, M_ERROR, 1, _("Can't update accurate hash disk ERR=%s\n"));
512 + if (!jcr->file_list->insert(jcr, fname, &elt)) {
513 + Jmsg(jcr, M_ERROR, 1, _("Can't update accurate hash ERR=%s\n"));
518 - /* we store CurFile, fname and ctime/mtime in the same chunk */
519 - item = (CurFile *)jcr->file_list->hash_malloc(sizeof(CurFile)+strlen(fname)+1);
520 - memcpy(item, &elt, sizeof(CurFile));
521 - item->fname = (char *)item+sizeof(CurFile);
522 - strcpy(item->fname, fname);
523 - jcr->file_list->insert(item->fname, item);
526 -// Dmsg2(dbglvl, "add fname=<%s> lstat=%s\n", fname, lstat);
527 + Dmsg2(dbglvl, "add fname=<%s> lstat=%s\n", fname, lstat);
532 fname = ff_pkt->fname;
535 - if (!accurate_lookup(jcr, fname, &elt)) {
536 + if (!jcr->file_list->lookup(jcr, fname, &elt)) {
537 Dmsg1(dbglvl, "accurate %s (not found)\n", fname);
544 - accurate_mark_file_as_seen(jcr, &elt);
545 + jcr->file_list->mark_as_seen(jcr, fname, &elt);
546 Dmsg2(dbglvl, "accurate %s = %i\n", fname, stat);
550 while (dir->recv() >= 0) {
551 len = strlen(dir->msg) + 1;
552 if (len < dir->msglen) {
553 - accurate_add_file(jcr, dir->msg, dir->msg + len);
554 + accurate_add_file(jcr, dir->msg, dir->msg + len);
566 - * Tokyo Cabinet library doesn't use smartalloc by default
567 - * results need to be released with real free()
570 -void realfree(void *p)
576 Index: src/filed/filed.h
577 ===================================================================
578 --- src/filed/filed.h (révision 7351)
579 +++ src/filed/filed.h (copie de travail)
583 #define FILE_DAEMON 1
584 -#ifdef USE_TCADB /* hash disk based */
587 -# include "lib/htable.h"
591 + * Used to store accurate information
601 + * Virtual class for accurate backend (libdb or htable)
603 +class AccurateBackend: public SMARTALLOC
606 + AccurateBackend() : SMARTALLOC() {}
607 + virtual ~AccurateBackend() {}
608 + virtual bool init(JCR *jcr, int nb_elt) = 0;
609 + virtual bool insert(JCR *jcr, char *key, CurFile *item) = 0;
610 + virtual bool lookup(JCR *jcr, char *key, CurFile *item) = 0;
611 + virtual bool mark_as_seen(JCR *jcr, char *key, CurFile *item) = 0;
612 + virtual CurFile *first(CurFile *elt) = 0;
613 + virtual CurFile *next() = 0;
614 + virtual void finish() = 0;
615 + virtual void destroy() {}
618 #include "filed_conf.h"
619 #include "fd_plugins.h"
620 #include "findlib/find.h"
621 Index: src/baconfig.h
622 ===================================================================
623 --- src/baconfig.h (révision 7351)
624 +++ src/baconfig.h (copie de travail)
627 #endif /* HAVE_WIN32 */
629 +/* Select db backend for accurate mode */
639 ===================================================================
640 --- src/jcr.h (révision 7351)
641 +++ src/jcr.h (copie de travail)
643 CRYPTO_CTX crypto; /* Crypto ctx */
644 DIRRES* director; /* Director resource */
645 bool VSS; /* VSS used by FD */
647 - TCADB *file_list; /* Previous file list (accurate mode) */
648 - POOLMEM *hash_name;
650 - htable *file_list; /* Previous file list (accurate mode) */
652 + AccurateBackend *file_list; /* Accurate backend store */
653 #endif /* FILE_DAEMON */