1 Index: src/filed/accurate.c
2 ===================================================================
3 --- src/filed/accurate.c (révision 7351)
4 +++ src/filed/accurate.c (copie de travail)
9 +#include "lib/htable.h"
10 +static int dbglvl=500;
12 -static int dbglvl=200;
16 -typedef struct PrivateCurFile {
20 - char *fname; /* not stored with tchdb mode */
27 -static void realfree(void *p); /* used by tokyo code */
30 - * Update hash element seen=1
31 + * This backend uses DB Berkeley
33 -static bool accurate_mark_file_as_seen(JCR *jcr, CurFile *elt)
34 +class AccurateBackendDB : public AccurateBackend
38 + ~AccurateBackendDB() { destroy(); }
39 + bool init(JCR *jcr, int nb_elt);
43 - if (!tcadbput(jcr->file_list,
44 - elt->fname, strlen(elt->fname)+1,
45 - elt, sizeof(CurFile)))
46 - { /* TODO: disabling accurate mode ? */
47 - Jmsg(jcr, M_ERROR, 1, _("Can't update accurate hash disk\n"));
49 + bool insert(JCR *jcr, char *key, CurFile *item);
50 + bool lookup(JCR *jcr, char *key, CurFile *item);
52 + bool mark_as_seen(JCR *jcr, char *key, CurFile *item);
54 + CurFile *first(CurFile *elt);
58 + DB *db; /* DB object */
59 + DB_ENV *dbenv; /* DB Env object */
60 + DBC *cursorp; /* DB cursor */
63 + POOLMEM *hash_name; /* file name for hash */
66 +void AccurateBackendDB::destroy()
68 + /* cleanup walk cursor if any */
70 + cursorp->c_close(cursorp);
76 + db->close(db, DB_NOSYNC);
80 + dbenv->close(dbenv, 0);
84 + unlink(hash_name); /* remove the hash on disk */
85 + free_pool_memory(hash_name);
90 -static bool accurate_lookup(JCR *jcr, char *fname, CurFile *ret)
91 +bool AccurateBackendDB::init(JCR *jcr, int nb_elt)
99 + Dmsg0(dbglvl, "init DB accurate backend\n");
101 - elt = (CurFile*)tcadbget(jcr->file_list,
102 - fname, strlen(fname)+1, &size);
104 + if ((ret = db_env_create(&dbenv, 0)) != 0) {
105 + Jmsg(jcr, M_ERROR, 1, _("Can't open initialize hash disk ERR=%i\n"), ret);
109 + dbenv->set_cachesize(dbenv, 0, 32 * 1024 * 1024, 0);
110 + dbenv->set_errfile(dbenv, stderr);
111 + dbenv->set_errpfx(dbenv, "hash");
113 + int flags = DB_PRIVATE | DB_INIT_LOCK | DB_INIT_MPOOL | DB_CREATE | DB_THREAD;
114 + if ((ret = dbenv->open(dbenv, working_directory, flags, 0)) != 0) {
115 + Jmsg(jcr, M_ERROR, 1, _("Can't open initialize hash disk ERR=%i\n"), ret);
116 + dbenv->close(dbenv, 0);
121 + dbenv->set_flags(dbenv, DB_TXN_NOSYNC, 1);
123 + if ((ret = db_create(&db, dbenv, 0)) != 0) {
124 + Jmsg(jcr, M_ERROR, 1, _("Can't open accurate hash disk ERR=%i\n"), ret);
128 + hash_name = get_pool_memory(PM_MESSAGE);
129 + make_unique_filename(&hash_name, jcr->JobId, "accurate");
131 + if ((ret = db->open(db,
132 + NULL, hash_name, NULL,
133 + DB_BTREE, DB_CREATE, 0600)) != 0)
135 - /* TODO: don't malloc/free results */
137 - elt->fname = fname;
138 - memcpy(ret, elt, sizeof(CurFile));
140 -// Dmsg1(dbglvl, "lookup <%s> ok\n", fname);
141 + db->close(db, DB_NOSYNC);
143 + free_pool_memory(hash_name);
145 + Jmsg(jcr, M_ERROR, 1, _("Can't setup hash disk ERR=%i\n"), ret);
153 -/* Create tokyo dbm hash file
154 - * If something goes wrong, we cancel accurate mode.
156 -static bool accurate_init(JCR *jcr, int nbfile)
157 +/* Just update the element->seen to know if we have seen it */
158 +bool AccurateBackendDB::mark_as_seen(JCR *jcr, char *key, CurFile *item)
160 - jcr->file_list = tcadbnew();
162 -// tchdbsetcache(jcr->file_list, 300000);
163 -// tchdbtune(jcr->file_list,
164 -// nbfile, /* nb bucket 0.5n to 4n */
165 -// 6, /* size of element 2^x */
167 -// 0); /* options like compression */
169 - jcr->hash_name = get_pool_memory(PM_MESSAGE);
170 - POOLMEM *temp = get_pool_memory(PM_MESSAGE);
172 + return insert(jcr, key, item);
175 - if (nbfile > 500000) {
176 - make_unique_filename(&jcr->hash_name, jcr->JobId, "accurate");
177 - pm_strcat(jcr->hash_name, ".tcb");
178 - Mmsg(temp, "%s#bnum=%i#mode=e#opts=l",
179 - jcr->hash_name, nbfile*4);
180 - Dmsg1(dbglvl, "Doing accurate hash on disk %s\n", jcr->hash_name);
182 - Dmsg0(dbglvl, "Doing accurate hash on memory\n");
183 - pm_strcpy(jcr->hash_name, "*");
184 - pm_strcpy(temp, "*");
185 +/* insert/replace */
186 +bool AccurateBackendDB::insert(JCR *jcr, char *key, CurFile *item)
189 + memset(&dbkey, 0, sizeof(DBT));
190 + memset(&dbdata, 0, sizeof(DBT));
192 + dbkey.size = strlen(key)+1;
193 + dbdata.data = item;
194 + dbdata.size = sizeof(CurFile);
195 + if ((ret = db->put(db, NULL, &dbkey, &dbdata, 0))) {
196 + Jmsg(jcr, M_ERROR, 1, _("Can't update accurate hash disk ERR=%i\n"), ret);
202 +bool AccurateBackendDB::lookup(JCR *jcr, char *key, CurFile *item)
206 + /* Zero out the DBTs before using them. */
207 + memset(&dbkey, 0, sizeof(DBT));
208 + memset(&dbdata, 0, sizeof(DBT));
211 + dbkey.size = strlen(key)+1;
213 - if(!tcadbopen(jcr->file_list, jcr->hash_name)){
214 - Jmsg(jcr, M_ERROR, 1, _("Can't open accurate hash disk\n"));
215 - Jmsg(jcr, M_INFO, 1, _("Disabling accurate mode\n"));
216 - tcadbdel(jcr->file_list);
217 - jcr->file_list = NULL;
218 - jcr->accurate = false;
219 + dbdata.data = item;
220 + dbdata.ulen = sizeof(CurFile);
221 + dbdata.flags = DB_DBT_USERMEM;
223 + if (db->get(db, NULL, &dbkey, &dbdata, 0) == 0) {
226 - free_pool_memory(temp);
227 - return jcr->file_list != NULL;
232 -/* This function is called at the end of backup
233 - * We walk over all hash disk element, and we check
236 + * We use user memory to copy data
238 -bool accurate_send_deleted_list(JCR *jcr)
239 +CurFile *AccurateBackendDB::first(CurFile *elt)
242 + /* Zero out the DBTs before using them. */
243 + memset(&dbkey, 0, sizeof(DBT));
244 + memset(&dbdata, 0, sizeof(DBT));
246 + dbdata.ulen = sizeof(CurFile);
247 + dbdata.flags = DB_DBT_USERMEM;
248 + db->cursor(db, NULL, &cursorp, 0);
252 +CurFile *AccurateBackendDB::next()
257 - int stream = STREAM_UNIX_ATTRIBUTES;
259 - if (!jcr->accurate || jcr->JobLevel == L_FULL) {
261 + if (cursorp->c_get(cursorp, &dbkey, &dbdata, DB_NEXT) == 0) {
262 + /* update fname field with fresh data */
263 + elt = (CurFile *)dbdata.data;
264 + elt->fname = (char *)dbkey.data;
271 - if (jcr->file_list == NULL) {
273 +void AccurateBackendDB::finish()
276 + cursorp->c_close(cursorp);
281 - ff_pkt = init_find_files();
282 - ff_pkt->type = FT_DELETED;
285 - /* traverse records */
286 - tcadbiterinit(jcr->file_list);
287 - while((key = tcadbiternext2(jcr->file_list)) != NULL) {
288 - elt = (CurFile *) tcadbget(jcr->file_list,
289 - key, strlen(key)+1, &size);
292 - if (!elt->seen) { /* already seen */
293 - ff_pkt->fname = key;
294 - ff_pkt->statp.st_mtime = elt->mtime;
295 - ff_pkt->statp.st_ctime = elt->ctime;
296 - encode_and_send_attributes(jcr, ff_pkt, stream);
300 - realfree(key); /* tokyo cabinet have to use real free() */
302 +/****************************************************************/
304 - term_find_files(ff_pkt);
306 - /* TODO: clean htable when this function is not reached ? */
307 - if (jcr->file_list) {
308 - if(!tcadbclose(jcr->file_list)){
309 - Jmsg(jcr, M_ERROR, 1, _("Can't close accurate hash disk\n"));
312 + * This is the htable implementation for accurate mode
314 +class AccurateBackendHT : public AccurateBackend
317 + ~AccurateBackendHT() { destroy(); }
318 + bool init(JCR *jcr, int nb_elt);
321 - /* delete the object */
322 - tcadbdel(jcr->file_list);
323 - if (!bstrcmp(jcr->hash_name, "*")) {
324 - unlink(jcr->hash_name);
326 + bool insert(JCR *jcr, char *key, CurFile *item);
327 + bool lookup(JCR *jcr, char *key, CurFile *item);
329 - free_pool_memory(jcr->hash_name);
330 - jcr->hash_name = NULL;
331 - jcr->file_list = NULL;
332 + bool mark_as_seen(JCR *jcr, char *key, CurFile *item);
334 + CurFile *first(CurFile *elt);
343 + hlink link; /* need this for htable link */
346 +void AccurateBackendHT::destroy()
355 +bool AccurateBackendHT::init(JCR *jcr, int nb_elt)
357 + Dmsg0(dbglvl, "init HT accurate backend\n");
359 + HTCurFile *elt=NULL;
360 + db = (htable *)malloc(sizeof(htable));
361 + db->init(elt, &elt->link, nb_elt);
365 -#else /* HTABLE mode */
366 +bool AccurateBackendHT::insert(JCR *jcr, char *key, CurFile *item)
368 + /* alloc CurFile + hlink + fname */
369 + HTCurFile *htf = (HTCurFile *)db->hash_malloc(sizeof(HTCurFile)+strlen(key)+1);
370 + memcpy(&htf->elt, item, sizeof(CurFile));
372 -static bool accurate_mark_file_as_seen(JCR *jcr, CurFile *elt)
373 + /* store fname at the end of the struct */
374 + htf->elt.fname = (char *) htf + sizeof(HTCurFile);
376 + strcpy(htf->elt.fname, key);
377 + db->insert(htf->elt.fname, htf);
381 +bool AccurateBackendHT::mark_as_seen(JCR *jcr, char *key, CurFile *item)
383 - CurFile *temp = (CurFile *)jcr->file_list->lookup(elt->fname);
384 - temp->seen = 1; /* records are in memory */
385 + HTCurFile *temp = (HTCurFile *)db->lookup(key);
387 + temp->elt.seen = 1; /* update in memory */
392 -static bool accurate_lookup(JCR *jcr, char *fname, CurFile *ret)
393 +bool AccurateBackendHT::lookup(JCR *jcr, char *key, CurFile *item)
398 - CurFile *temp = (CurFile *)jcr->file_list->lookup(fname);
399 + HTCurFile *temp = (HTCurFile *)db->lookup(key);
401 - memcpy(ret, temp, sizeof(CurFile));
402 + memcpy(item, &temp->elt, sizeof(CurFile));
404 -// Dmsg1(dbglvl, "lookup <%s> ok\n", fname);
410 +CurFile *AccurateBackendHT::first(CurFile *elt)
412 + HTCurFile *temp = (HTCurFile *)db->first();
416 +CurFile *AccurateBackendHT::next()
418 + HTCurFile *temp = (HTCurFile *)db->next();
422 +void AccurateBackendHT::finish()
426 +/****************************************************************/
429 + * For less than 1M files, use htable in memory
431 static bool accurate_init(JCR *jcr, int nbfile)
434 - jcr->file_list = (htable *)malloc(sizeof(htable));
435 - jcr->file_list->init(elt, &elt->link, nbfile);
437 + if (nbfile > 1000000) {
438 + jcr->file_list = New(AccurateBackendDB);
440 + jcr->file_list = New(AccurateBackendHT);
443 + jcr->file_list = New(AccurateBackendHT);
445 + jcr->file_list->init(jcr, nbfile);
452 bool accurate_send_deleted_list(JCR *jcr)
458 int stream = STREAM_UNIX_ATTRIBUTES;
460 @@ -242,31 +360,28 @@
462 ff_pkt = init_find_files();
463 ff_pkt->type = FT_DELETED;
465 - foreach_htable (elt, jcr->file_list) {
466 - if (!elt->seen) { /* already seen */
467 - Dmsg2(dbglvl, "deleted fname=%s seen=%i\n", elt->fname, elt->seen);
468 - ff_pkt->fname = elt->fname;
469 - ff_pkt->statp.st_mtime = elt->mtime;
470 - ff_pkt->statp.st_ctime = elt->ctime;
472 + for (item = jcr->file_list->first(&elt); item ; item = jcr->file_list->next()) {
473 + if (!item->seen) { /* already seen */
474 + Dmsg2(dbglvl, "deleted fname=%s seen=%i\n", item->fname, item->seen);
475 + ff_pkt->fname = item->fname;
476 + ff_pkt->statp.st_mtime = item->mtime;
477 + ff_pkt->statp.st_ctime = item->ctime;
478 encode_and_send_attributes(jcr, ff_pkt, stream);
480 -// free(elt->fname);
482 + jcr->file_list->finish();
484 term_find_files(ff_pkt);
486 /* TODO: clean htable when this function is not reached ? */
487 if (jcr->file_list) {
488 - jcr->file_list->destroy();
489 - free(jcr->file_list);
490 + delete jcr->file_list;
491 jcr->file_list = NULL;
496 -#endif /* common code */
498 static bool accurate_add_file(JCR *jcr, char *fname, char *lstat)
501 @@ -278,25 +393,12 @@
502 elt.mtime = statp.st_mtime;
506 - if (!tcadbput(jcr->file_list,
507 - fname, strlen(fname)+1,
508 - &elt, sizeof(CurFile)))
510 - Jmsg(jcr, M_ERROR, 1, _("Can't update accurate hash disk ERR=%s\n"));
511 + if (!jcr->file_list->insert(jcr, fname, &elt)) {
512 + Jmsg(jcr, M_ERROR, 1, _("Can't update accurate hash ERR=%s\n"));
517 - /* we store CurFile, fname and ctime/mtime in the same chunk */
518 - item = (CurFile *)jcr->file_list->hash_malloc(sizeof(CurFile)+strlen(fname)+1);
519 - memcpy(item, &elt, sizeof(CurFile));
520 - item->fname = (char *)item+sizeof(CurFile);
521 - strcpy(item->fname, fname);
522 - jcr->file_list->insert(item->fname, item);
525 -// Dmsg2(dbglvl, "add fname=<%s> lstat=%s\n", fname, lstat);
526 + Dmsg2(dbglvl, "add fname=<%s> lstat=%s\n", fname, lstat);
531 fname = ff_pkt->fname;
534 - if (!accurate_lookup(jcr, fname, &elt)) {
535 + if (!jcr->file_list->lookup(jcr, fname, &elt)) {
536 Dmsg1(dbglvl, "accurate %s (not found)\n", fname);
543 - accurate_mark_file_as_seen(jcr, &elt);
544 + jcr->file_list->mark_as_seen(jcr, fname, &elt);
545 Dmsg2(dbglvl, "accurate %s = %i\n", fname, stat);
549 while (dir->recv() >= 0) {
550 len = strlen(dir->msg) + 1;
551 if (len < dir->msglen) {
552 - accurate_add_file(jcr, dir->msg, dir->msg + len);
553 + accurate_add_file(jcr, dir->msg, dir->msg + len);
565 - * Tokyo Cabinet library doesn't use smartalloc by default
566 - * results need to be released with real free()
569 -void realfree(void *p)
575 Index: src/filed/filed.h
576 ===================================================================
577 --- src/filed/filed.h (révision 7351)
578 +++ src/filed/filed.h (copie de travail)
582 #define FILE_DAEMON 1
583 -#ifdef USE_TCADB /* hash disk based */
586 -# include "lib/htable.h"
590 + * Used to store accurate information
600 + * Virtual class for accurate backend (libdb or htable)
602 +class AccurateBackend: public SMARTALLOC
605 + AccurateBackend() : SMARTALLOC() {}
606 + virtual ~AccurateBackend() {}
607 + virtual bool init(JCR *jcr, int nb_elt) = 0;
608 + virtual bool insert(JCR *jcr, char *key, CurFile *item) = 0;
609 + virtual bool lookup(JCR *jcr, char *key, CurFile *item) = 0;
610 + virtual bool mark_as_seen(JCR *jcr, char *key, CurFile *item) = 0;
611 + virtual CurFile *first(CurFile *elt) = 0;
612 + virtual CurFile *next() = 0;
613 + virtual void finish() = 0;
614 + virtual void destroy() {}
617 #include "filed_conf.h"
618 #include "fd_plugins.h"
619 #include "findlib/find.h"
620 Index: src/baconfig.h
621 ===================================================================
622 --- src/baconfig.h (révision 7351)
623 +++ src/baconfig.h (copie de travail)
626 #endif /* HAVE_WIN32 */
628 +/* Select db backend for accurate mode */
638 ===================================================================
639 --- src/jcr.h (révision 7351)
640 +++ src/jcr.h (copie de travail)
642 CRYPTO_CTX crypto; /* Crypto ctx */
643 DIRRES* director; /* Director resource */
644 bool VSS; /* VSS used by FD */
646 - TCADB *file_list; /* Previous file list (accurate mode) */
647 - POOLMEM *hash_name;
649 - htable *file_list; /* Previous file list (accurate mode) */
651 + AccurateBackend *file_list; /* Accurate backend store */
652 #endif /* FILE_DAEMON */