1 Index: src/filed/accurate.c
2 ===================================================================
3 --- src/filed/accurate.c (révision 7288)
4 +++ src/filed/accurate.c (copie de travail)
9 +#include "lib/htable.h"
10 +static int dbglvl=500;
12 -static int dbglvl=200;
16 -typedef struct PrivateCurFile {
20 - char *fname; /* not stored with tchdb mode */
27 -static void realfree(void *p); /* used by tokyo code */
30 - * Update hash element seen=1
31 + * This backend uses DB Berkeley
33 -static bool accurate_mark_file_as_seen(JCR *jcr, CurFile *elt)
34 +class AccurateBackendDB : public AccurateBackend
38 + ~AccurateBackendDB() { destroy(); }
39 + bool init(JCR *jcr, int nb_elt);
43 - if (!tcadbput(jcr->file_list,
44 - elt->fname, strlen(elt->fname)+1,
45 - elt, sizeof(CurFile)))
46 - { /* TODO: disabling accurate mode ? */
47 - Jmsg(jcr, M_ERROR, 1, _("Can't update accurate hash disk\n"));
49 + bool insert(JCR *jcr, char *key, CurFile *item);
50 + bool lookup(JCR *jcr, char *key, CurFile *item);
52 + bool mark_as_seen(JCR *jcr, char *key, CurFile *item);
54 + CurFile *first(CurFile *elt);
58 + DB *db; /* DB object */
59 + DBC *cursorp; /* DB cursor */
62 + POOLMEM *hash_name; /* file name for hash */
65 +void AccurateBackendDB::destroy()
67 + /* cleanup walk cursor if any */
69 + cursorp->c_close(cursorp);
75 + db->close(db, DB_NOSYNC);
79 + unlink(hash_name); /* remove the hash on disk */
80 + free_pool_memory(hash_name);
85 -static bool accurate_lookup(JCR *jcr, char *fname, CurFile *ret)
86 +bool AccurateBackendDB::init(JCR *jcr, int nb_elt)
94 + Dmsg0(dbglvl, "init DB accurate backend\n");
96 - elt = (CurFile*)tcadbget(jcr->file_list,
97 - fname, strlen(fname)+1, &size);
99 + if ((ret = db_create(&db, NULL, 0)) != 0) {
100 + Jmsg(jcr, M_ERROR, 1, _("Can't open accurate hash disk ERR=%i\n"), ret);
104 + db->set_errfile(db, stderr);
105 + db->set_errpfx(db, "hash");
107 + if ((ret = db->set_cachesize(db, 0, 32 * 1024 * 1024, 0)) != 0) {
108 + Jmsg(jcr, M_ERROR, 1, _("Can't setup hash disk cache size ERR=%i\n"), ret);
109 + db->close(db, DB_NOSYNC);
113 + hash_name = get_pool_memory(PM_MESSAGE);
114 + make_unique_filename(&hash_name, jcr->JobId, "accurate");
116 + if ((ret = db->open(db,
117 + NULL, hash_name, NULL,
118 + DB_BTREE, DB_CREATE, 0600)) != 0)
120 - /* TODO: don't malloc/free results */
122 - elt->fname = fname;
123 - memcpy(ret, elt, sizeof(CurFile));
125 -// Dmsg1(dbglvl, "lookup <%s> ok\n", fname);
126 + db->close(db, DB_NOSYNC);
128 + free_pool_memory(hash_name);
130 + Jmsg(jcr, M_ERROR, 1, _("Can't setup hash disk ERR=%i\n"), ret);
138 -/* Create tokyo dbm hash file
139 - * If something goes wrong, we cancel accurate mode.
141 -static bool accurate_init(JCR *jcr, int nbfile)
142 +/* Just update the element->seen to know if we have seen it */
143 +bool AccurateBackendDB::mark_as_seen(JCR *jcr, char *key, CurFile *item)
145 - jcr->file_list = tcadbnew();
147 -// tchdbsetcache(jcr->file_list, 300000);
148 -// tchdbtune(jcr->file_list,
149 -// nbfile, /* nb bucket 0.5n to 4n */
150 -// 6, /* size of element 2^x */
152 -// 0); /* options like compression */
154 - jcr->hash_name = get_pool_memory(PM_MESSAGE);
155 - POOLMEM *temp = get_pool_memory(PM_MESSAGE);
157 + return insert(jcr, key, item);
160 - if (nbfile > 500000) {
161 - make_unique_filename(&jcr->hash_name, jcr->JobId, "accurate");
162 - pm_strcat(jcr->hash_name, ".tcb");
163 - Mmsg(temp, "%s#bnum=%i#mode=e#opts=l",
164 - jcr->hash_name, nbfile*4);
165 - Dmsg1(dbglvl, "Doing accurate hash on disk %s\n", jcr->hash_name);
167 - Dmsg0(dbglvl, "Doing accurate hash on memory\n");
168 - pm_strcpy(jcr->hash_name, "*");
169 - pm_strcpy(temp, "*");
170 +/* insert/replace */
171 +bool AccurateBackendDB::insert(JCR *jcr, char *key, CurFile *item)
174 + memset(&dbkey, 0, sizeof(DBT));
175 + memset(&dbdata, 0, sizeof(DBT));
177 + dbkey.size = strlen(key)+1;
178 + dbdata.data = item;
179 + dbdata.size = sizeof(CurFile);
180 + if ((ret = db->put(db, NULL, &dbkey, &dbdata, 0))) {
181 + Jmsg(jcr, M_ERROR, 1, _("Can't update accurate hash disk ERR=%i\n"), ret);
187 +bool AccurateBackendDB::lookup(JCR *jcr, char *key, CurFile *item)
191 + /* Zero out the DBTs before using them. */
192 + memset(&dbkey, 0, sizeof(DBT));
193 + memset(&dbdata, 0, sizeof(DBT));
196 + dbkey.size = strlen(key)+1;
198 - if(!tcadbopen(jcr->file_list, jcr->hash_name)){
199 - Jmsg(jcr, M_ERROR, 1, _("Can't open accurate hash disk\n"));
200 - Jmsg(jcr, M_INFO, 1, _("Disabling accurate mode\n"));
201 - tcadbdel(jcr->file_list);
202 - jcr->file_list = NULL;
203 - jcr->accurate = false;
204 + dbdata.data = item;
205 + dbdata.ulen = sizeof(CurFile);
206 + dbdata.flags = DB_DBT_USERMEM;
208 + if (db->get(db, NULL, &dbkey, &dbdata, 0) == 0) {
211 - free_pool_memory(temp);
212 - return jcr->file_list != NULL;
217 -/* This function is called at the end of backup
218 - * We walk over all hash disk element, and we check
221 + * We use user memory to copy data
223 -bool accurate_send_deleted_list(JCR *jcr)
224 +CurFile *AccurateBackendDB::first(CurFile *elt)
227 + /* Zero out the DBTs before using them. */
228 + memset(&dbkey, 0, sizeof(DBT));
229 + memset(&dbdata, 0, sizeof(DBT));
231 + dbdata.ulen = sizeof(CurFile);
232 + dbdata.flags = DB_DBT_USERMEM;
233 + db->cursor(db, NULL, &cursorp, 0);
237 +CurFile *AccurateBackendDB::next()
242 - int stream = STREAM_UNIX_ATTRIBUTES;
244 - if (!jcr->accurate || jcr->JobLevel == L_FULL) {
246 + if (cursorp->c_get(cursorp, &dbkey, &dbdata, DB_NEXT) == 0) {
247 + /* update fname field with fresh data */
248 + elt = (CurFile *)dbdata.data;
249 + elt->fname = (char *)dbkey.data;
256 - if (jcr->file_list == NULL) {
258 +void AccurateBackendDB::finish()
261 + cursorp->c_close(cursorp);
266 - ff_pkt = init_find_files();
267 - ff_pkt->type = FT_DELETED;
270 - /* traverse records */
271 - tcadbiterinit(jcr->file_list);
272 - while((key = tcadbiternext2(jcr->file_list)) != NULL) {
273 - elt = (CurFile *) tcadbget(jcr->file_list,
274 - key, strlen(key)+1, &size);
277 - if (!elt->seen) { /* already seen */
278 - ff_pkt->fname = key;
279 - ff_pkt->statp.st_mtime = elt->mtime;
280 - ff_pkt->statp.st_ctime = elt->ctime;
281 - encode_and_send_attributes(jcr, ff_pkt, stream);
285 - realfree(key); /* tokyo cabinet have to use real free() */
287 +/****************************************************************/
289 - term_find_files(ff_pkt);
291 - /* TODO: clean htable when this function is not reached ? */
292 - if (jcr->file_list) {
293 - if(!tcadbclose(jcr->file_list)){
294 - Jmsg(jcr, M_ERROR, 1, _("Can't close accurate hash disk\n"));
297 + * This is the htable implementation for accurate mode
299 +class AccurateBackendHT : public AccurateBackend
302 + ~AccurateBackendHT() { destroy(); }
303 + bool init(JCR *jcr, int nb_elt);
306 - /* delete the object */
307 - tcadbdel(jcr->file_list);
308 - if (!bstrcmp(jcr->hash_name, "*")) {
309 - unlink(jcr->hash_name);
311 + bool insert(JCR *jcr, char *key, CurFile *item);
312 + bool lookup(JCR *jcr, char *key, CurFile *item);
314 - free_pool_memory(jcr->hash_name);
315 - jcr->hash_name = NULL;
316 - jcr->file_list = NULL;
317 + bool mark_as_seen(JCR *jcr, char *key, CurFile *item);
319 + CurFile *first(CurFile *elt);
328 + hlink link; /* need this for htable link */
331 +void AccurateBackendHT::destroy()
340 +bool AccurateBackendHT::init(JCR *jcr, int nb_elt)
342 + Dmsg0(dbglvl, "init HT accurate backend\n");
344 + HTCurFile *elt=NULL;
345 + db = (htable *)malloc(sizeof(htable));
346 + db->init(elt, &elt->link, nb_elt);
350 -#else /* HTABLE mode */
351 +bool AccurateBackendHT::insert(JCR *jcr, char *key, CurFile *item)
353 + /* alloc CurFile + hlink + fname */
354 + HTCurFile *htf = (HTCurFile *)db->hash_malloc(sizeof(HTCurFile)+strlen(key)+1);
355 + memcpy(&htf->elt, item, sizeof(CurFile));
357 -static bool accurate_mark_file_as_seen(JCR *jcr, CurFile *elt)
358 + /* store fname at the end of the struct */
359 + htf->elt.fname = (char *) htf + sizeof(HTCurFile);
361 + strcpy(htf->elt.fname, key);
362 + db->insert(htf->elt.fname, htf);
366 +bool AccurateBackendHT::mark_as_seen(JCR *jcr, char *key, CurFile *item)
368 - CurFile *temp = (CurFile *)jcr->file_list->lookup(elt->fname);
369 - temp->seen = 1; /* records are in memory */
370 + HTCurFile *temp = (HTCurFile *)db->lookup(key);
372 + temp->elt.seen = 1; /* update in memory */
377 -static bool accurate_lookup(JCR *jcr, char *fname, CurFile *ret)
378 +bool AccurateBackendHT::lookup(JCR *jcr, char *key, CurFile *item)
383 - CurFile *temp = (CurFile *)jcr->file_list->lookup(fname);
384 + HTCurFile *temp = (HTCurFile *)db->lookup(key);
386 - memcpy(ret, temp, sizeof(CurFile));
387 + memcpy(item, &temp->elt, sizeof(CurFile));
389 -// Dmsg1(dbglvl, "lookup <%s> ok\n", fname);
395 +CurFile *AccurateBackendHT::first(CurFile *elt)
397 + HTCurFile *temp = (HTCurFile *)db->first();
401 +CurFile *AccurateBackendHT::next()
403 + HTCurFile *temp = (HTCurFile *)db->next();
407 +void AccurateBackendHT::finish()
411 +/****************************************************************/
414 + * For less than 1M files, use htable in memory
416 static bool accurate_init(JCR *jcr, int nbfile)
419 - jcr->file_list = (htable *)malloc(sizeof(htable));
420 - jcr->file_list->init(elt, &elt->link, nbfile);
422 + if (nbfile > 1000000) {
423 + jcr->file_list = New(AccurateBackendDB);
425 + jcr->file_list = New(AccurateBackendHT);
428 + jcr->file_list = New(AccurateBackendHT);
430 + jcr->file_list->init(jcr, nbfile);
437 bool accurate_send_deleted_list(JCR *jcr)
443 int stream = STREAM_UNIX_ATTRIBUTES;
445 @@ -242,31 +345,28 @@
447 ff_pkt = init_find_files();
448 ff_pkt->type = FT_DELETED;
450 - foreach_htable (elt, jcr->file_list) {
451 - if (!elt->seen) { /* already seen */
452 - Dmsg2(dbglvl, "deleted fname=%s seen=%i\n", elt->fname, elt->seen);
453 - ff_pkt->fname = elt->fname;
454 - ff_pkt->statp.st_mtime = elt->mtime;
455 - ff_pkt->statp.st_ctime = elt->ctime;
457 + for (item = jcr->file_list->first(&elt); item ; item = jcr->file_list->next()) {
458 + if (!item->seen) { /* already seen */
459 + Dmsg2(dbglvl, "deleted fname=%s seen=%i\n", item->fname, item->seen);
460 + ff_pkt->fname = item->fname;
461 + ff_pkt->statp.st_mtime = item->mtime;
462 + ff_pkt->statp.st_ctime = item->ctime;
463 encode_and_send_attributes(jcr, ff_pkt, stream);
465 -// free(elt->fname);
467 + jcr->file_list->finish();
469 term_find_files(ff_pkt);
471 /* TODO: clean htable when this function is not reached ? */
472 if (jcr->file_list) {
473 - jcr->file_list->destroy();
474 - free(jcr->file_list);
475 + delete jcr->file_list;
476 jcr->file_list = NULL;
481 -#endif /* common code */
483 static bool accurate_add_file(JCR *jcr, char *fname, char *lstat)
486 @@ -278,25 +378,12 @@
487 elt.mtime = statp.st_mtime;
491 - if (!tcadbput(jcr->file_list,
492 - fname, strlen(fname)+1,
493 - &elt, sizeof(CurFile)))
495 - Jmsg(jcr, M_ERROR, 1, _("Can't update accurate hash disk ERR=%s\n"));
496 + if (!jcr->file_list->insert(jcr, fname, &elt)) {
497 + Jmsg(jcr, M_ERROR, 1, _("Can't update accurate hash ERR=%s\n"));
502 - /* we store CurFile, fname and ctime/mtime in the same chunk */
503 - item = (CurFile *)jcr->file_list->hash_malloc(sizeof(CurFile)+strlen(fname)+1);
504 - memcpy(item, &elt, sizeof(CurFile));
505 - item->fname = (char *)item+sizeof(CurFile);
506 - strcpy(item->fname, fname);
507 - jcr->file_list->insert(item->fname, item);
510 -// Dmsg2(dbglvl, "add fname=<%s> lstat=%s\n", fname, lstat);
511 + Dmsg2(dbglvl, "add fname=<%s> lstat=%s\n", fname, lstat);
516 fname = ff_pkt->fname;
519 - if (!accurate_lookup(jcr, fname, &elt)) {
520 + if (!jcr->file_list->lookup(jcr, fname, &elt)) {
521 Dmsg1(dbglvl, "accurate %s (not found)\n", fname);
528 - accurate_mark_file_as_seen(jcr, &elt);
529 + jcr->file_list->mark_as_seen(jcr, fname, &elt);
530 Dmsg2(dbglvl, "accurate %s = %i\n", fname, stat);
534 while (dir->recv() >= 0) {
535 len = strlen(dir->msg) + 1;
536 if (len < dir->msglen) {
537 - accurate_add_file(jcr, dir->msg, dir->msg + len);
538 + accurate_add_file(jcr, dir->msg, dir->msg + len);
550 - * Tokyo Cabinet library doesn't use smartalloc by default
551 - * results need to be released with real free()
554 -void realfree(void *p)
560 Index: src/filed/filed.h
561 ===================================================================
562 --- src/filed/filed.h (révision 7288)
563 +++ src/filed/filed.h (copie de travail)
567 #define FILE_DAEMON 1
568 -#ifdef USE_TCADB /* hash disk based */
571 -# include "lib/htable.h"
575 + * Used to store accurate information
585 + * Virtual class for accurate backend (libdb or htable)
587 +class AccurateBackend: public SMARTALLOC
590 + AccurateBackend() : SMARTALLOC() {}
591 + virtual ~AccurateBackend() {}
592 + virtual bool init(JCR *jcr, int nb_elt) = 0;
593 + virtual bool insert(JCR *jcr, char *key, CurFile *item) = 0;
594 + virtual bool lookup(JCR *jcr, char *key, CurFile *item) = 0;
595 + virtual bool mark_as_seen(JCR *jcr, char *key, CurFile *item) = 0;
596 + virtual CurFile *first(CurFile *elt) = 0;
597 + virtual CurFile *next() = 0;
598 + virtual void finish() = 0;
599 + virtual void destroy() {}
602 #include "filed_conf.h"
603 #include "fd_plugins.h"
604 #include "findlib/find.h"
605 Index: src/baconfig.h
606 ===================================================================
607 --- src/baconfig.h (révision 7288)
608 +++ src/baconfig.h (copie de travail)
611 #endif /* HAVE_WIN32 */
613 +/* Select db backend for accurate mode */
623 ===================================================================
624 --- src/jcr.h (révision 7288)
625 +++ src/jcr.h (copie de travail)
627 CRYPTO_CTX crypto; /* Crypto ctx */
628 DIRRES* director; /* Director resource */
629 bool VSS; /* VSS used by FD */
631 - TCADB *file_list; /* Previous file list (accurate mode) */
632 - POOLMEM *hash_name;
634 - htable *file_list; /* Previous file list (accurate mode) */
636 + AccurateBackend *file_list; /* Accurate backend store */
637 #endif /* FILE_DAEMON */