X-Git-Url: https://git.sur5r.net/?a=blobdiff_plain;f=bacula%2Fsrc%2Ffiled%2Faccurate.c;h=e404d18f5e03c9190e3664d77db58e7caf1e2952;hb=f5571f4e4916261503020df01b6f9d30d17ae194;hp=c75d785d66d9854bcba49f9a2346ddd09ba35451;hpb=5f92612551bf94a1ab52b5dd5d16850e560a5696;p=bacula%2Fbacula diff --git a/bacula/src/filed/accurate.c b/bacula/src/filed/accurate.c index c75d785d66..e404d18f5e 100644 --- a/bacula/src/filed/accurate.c +++ b/bacula/src/filed/accurate.c @@ -33,8 +33,10 @@ #include "bacula.h" #include "filed.h" +static int dbglvl=200; + typedef struct PrivateCurFile { -#ifndef USE_TCHDB +#ifndef USE_TCADB hlink link; #endif char *fname; /* not stored with tchdb mode */ @@ -43,19 +45,23 @@ typedef struct PrivateCurFile { bool seen; } CurFile; -#ifdef USE_TCHDB +#ifdef USE_TCADB +static void realfree(void *p); /* used by tokyo code */ +/* + * Update hash element seen=1 + */ static bool accurate_mark_file_as_seen(JCR *jcr, CurFile *elt) { bool ret=true; elt->seen = 1; - if (!tchdbputasync(jcr->file_list, - elt->fname, strlen(elt->fname)+1, - elt, sizeof(CurFile))) - { - Dmsg1(2, "can't update <%s>\n", elt->fname); - ret = false; /* TODO: add error message */ + if (!tcadbput(jcr->file_list, + elt->fname, strlen(elt->fname)+1, + elt, sizeof(CurFile))) + { /* TODO: disabling accurate mode ? */ + Jmsg(jcr, M_ERROR, 1, _("Can't update accurate hash disk\n")); + ret = false; } return ret; @@ -65,48 +71,72 @@ static bool accurate_lookup(JCR *jcr, char *fname, CurFile *ret) { bool found=false; ret->seen = 0; + int size; + CurFile *elt; - if (tchdbget3(jcr->file_list, - fname, strlen(fname)+1, - ret, sizeof(CurFile)) != -1) + elt = (CurFile*)tcadbget(jcr->file_list, + fname, strlen(fname)+1, &size); + if (elt) { + /* TODO: don't malloc/free results */ found = true; - ret->fname = fname; + elt->fname = fname; + memcpy(ret, elt, sizeof(CurFile)); + realfree(elt); +// Dmsg1(dbglvl, "lookup <%s> ok\n", fname); } - - if (found) { - Dmsg1(2, "lookup <%s> ok\n", fname); - } - return found; } +/* Create tokyo dbm hash file + * If something goes wrong, we cancel accurate mode. + */ static bool accurate_init(JCR *jcr, int nbfile) { - jcr->file_list = tchdbnew(); - tchdbsetcache(jcr->file_list, 300000); - tchdbtune(jcr->file_list, - nbfile, /* nb bucket 0.5n to 4n */ - 6, /* size of element 2^x */ - 16, - 0); /* options like compression */ - /* TODO: make accurate file unique */ - POOLMEM *name = get_pool_memory(PM_MESSAGE); - make_unique_filename(name, jcr->JobId, "accurate"); - - if(!tchdbopen(jcr->file_list, name, HDBOWRITER | HDBOCREAT)){ - /* TODO: handle error creation */ - //ecode = tchdbecode(hdb); - //fprintf(stderr, "open error: %s\n", tchdberrmsg(ecode)); + jcr->file_list = tcadbnew(); +// +// tchdbsetcache(jcr->file_list, 300000); +// tchdbtune(jcr->file_list, +// nbfile, /* nb bucket 0.5n to 4n */ +// 6, /* size of element 2^x */ +// 16, +// 0); /* options like compression */ +// + jcr->hash_name = get_pool_memory(PM_MESSAGE); + POOLMEM *temp = get_pool_memory(PM_MESSAGE); + + if (nbfile > 500000) { + make_unique_filename(&jcr->hash_name, jcr->JobId, "accurate"); + pm_strcat(jcr->hash_name, ".tcb"); + Mmsg(temp, "%s#bnum=%i#mode=e#opts=l", + jcr->hash_name, nbfile*4); + Dmsg1(dbglvl, "Doing accurate hash on disk %s\n", jcr->hash_name); + } else { + Dmsg0(dbglvl, "Doing accurate hash on memory\n"); + pm_strcpy(jcr->hash_name, "*"); + pm_strcpy(temp, "*"); } - free_pool_memory(name); - return true; + + if(!tcadbopen(jcr->file_list, jcr->hash_name)){ + Jmsg(jcr, M_ERROR, 1, _("Can't open accurate hash disk\n")); + Jmsg(jcr, M_INFO, 1, _("Disabling accurate mode\n")); + tcadbdel(jcr->file_list); + jcr->file_list = NULL; + jcr->accurate = false; + } + free_pool_memory(temp); + return jcr->file_list != NULL; } - +/* This function is called at the end of backup + * We walk over all hash disk element, and we check + * for elt.seen. + */ bool accurate_send_deleted_list(JCR *jcr) { - CurFile elt; + char *key; + CurFile *elt; + int size; FF_PKT *ff_pkt; int stream = STREAM_UNIX_ATTRIBUTES; @@ -121,47 +151,40 @@ bool accurate_send_deleted_list(JCR *jcr) ff_pkt = init_find_files(); ff_pkt->type = FT_DELETED; - char *key; - /* traverse records */ - tchdbiterinit(jcr->file_list); - while((key = tchdbiternext2(jcr->file_list)) != NULL) { - if (tchdbget3(jcr->file_list, - key, strlen(key)+1, - &elt, sizeof(CurFile)) != -1) + /* traverse records */ + tcadbiterinit(jcr->file_list); + while((key = tcadbiternext2(jcr->file_list)) != NULL) { + elt = (CurFile *) tcadbget(jcr->file_list, + key, strlen(key)+1, &size); + if (elt) { - if (!elt.seen) { + if (!elt->seen) { /* already seen */ ff_pkt->fname = key; - ff_pkt->statp.st_mtime = elt.mtime; - ff_pkt->statp.st_ctime = elt.ctime; + ff_pkt->statp.st_mtime = elt->mtime; + ff_pkt->statp.st_ctime = elt->ctime; encode_and_send_attributes(jcr, ff_pkt, stream); - Dmsg1(2, "deleted <%s>\n", key); } -// free(key); - - } else { /* TODO: add error message */ - Dmsg1(2, "No value for <%s> key\n", key); + realfree(elt); } + realfree(key); /* tokyo cabinet have to use real free() */ } term_find_files(ff_pkt); bail_out: /* TODO: clean htable when this function is not reached ? */ if (jcr->file_list) { - if(!tchdbclose(jcr->file_list)){ -// ecode = tchdbecode(hdb); -// fprintf(stderr, "close error: %s\n", tchdberrmsg(ecode)); + if(!tcadbclose(jcr->file_list)){ + Jmsg(jcr, M_ERROR, 1, _("Can't close accurate hash disk\n")); } /* delete the object */ - tchdbdel(jcr->file_list); - - POOLMEM *name = get_pool_memory(PM_MESSAGE); - make_unique_filename(name, jcr->JobId, "accurate"); - -// unlink(name); - - free_pool_memory(name); + tcadbdel(jcr->file_list); + if (!bstrcmp(jcr->hash_name, "*")) { + unlink(jcr->hash_name); + } + free_pool_memory(jcr->hash_name); + jcr->hash_name = NULL; jcr->file_list = NULL; } return true; @@ -185,10 +208,7 @@ static bool accurate_lookup(JCR *jcr, char *fname, CurFile *ret) if (temp) { memcpy(ret, temp, sizeof(CurFile)); found=true; - } - - if (found) { - Dmsg1(2, "lookup <%s> ok\n", fname); +// Dmsg1(dbglvl, "lookup <%s> ok\n", fname); } return found; @@ -202,6 +222,10 @@ static bool accurate_init(JCR *jcr, int nbfile) return true; } +/* This function is called at the end of backup + * We walk over all hash disk element, and we check + * for elt.seen. + */ bool accurate_send_deleted_list(JCR *jcr) { CurFile *elt; @@ -221,7 +245,7 @@ bool accurate_send_deleted_list(JCR *jcr) foreach_htable (elt, jcr->file_list) { if (!elt->seen) { /* already seen */ - Dmsg2(1, "deleted fname=%s seen=%i\n", elt->fname, elt->seen); + Dmsg2(dbglvl, "deleted fname=%s seen=%i\n", elt->fname, elt->seen); ff_pkt->fname = elt->fname; ff_pkt->statp.st_mtime = elt->mtime; ff_pkt->statp.st_ctime = elt->ctime; @@ -241,10 +265,11 @@ bail_out: return true; } -#endif +#endif /* common code */ static bool accurate_add_file(JCR *jcr, char *fname, char *lstat) { + bool ret = true; CurFile elt; struct stat statp; int LinkFIc; @@ -253,13 +278,13 @@ static bool accurate_add_file(JCR *jcr, char *fname, char *lstat) elt.mtime = statp.st_mtime; elt.seen = 0; -#ifdef USE_TCHDB - if (!tchdbputasync(jcr->file_list, - fname, strlen(fname)+1, - &elt, sizeof(CurFile))) +#ifdef USE_TCADB + if (!tcadbput(jcr->file_list, + fname, strlen(fname)+1, + &elt, sizeof(CurFile))) { - Dmsg1(2, "Can't add <%s> to file_list\n", fname); - /* TODO: check error */ + Jmsg(jcr, M_ERROR, 1, _("Can't update accurate hash disk ERR=%s\n")); + ret = false; } #else /* HTABLE */ CurFile *item; @@ -271,8 +296,8 @@ static bool accurate_add_file(JCR *jcr, char *fname, char *lstat) jcr->file_list->insert(item->fname, item); #endif - Dmsg2(2, "add fname=<%s> lstat=%s\n", fname, lstat); - return true; +// Dmsg2(dbglvl, "add fname=<%s> lstat=%s\n", fname, lstat); + return ret; } /* @@ -300,13 +325,13 @@ bool accurate_check_file(JCR *jcr, FF_PKT *ff_pkt) } if (!accurate_lookup(jcr, fname, &elt)) { - Dmsg1(2, "accurate %s (not found)\n", fname); + Dmsg1(dbglvl, "accurate %s (not found)\n", fname); stat = true; goto bail_out; } if (elt.seen) { /* file has been seen ? */ - Dmsg1(2, "accurate %s (already seen)\n", fname); + Dmsg1(dbglvl, "accurate %s (already seen)\n", fname); goto bail_out; } @@ -318,13 +343,8 @@ bool accurate_check_file(JCR *jcr, FF_PKT *ff_pkt) stat = true; } - if (stat) { - Dmsg4(1, "%i = %i\t%i = %i\n", elt.mtime, ff_pkt->statp.st_mtime, - elt.ctime, ff_pkt->statp.st_ctime); - } - accurate_mark_file_as_seen(jcr, &elt); - Dmsg2(2, "accurate %s = %i\n", fname, stat); + Dmsg2(dbglvl, "accurate %s = %i\n", fname, stat); bail_out: unstrip_path(ff_pkt); @@ -348,7 +368,6 @@ int accurate_cmd(JCR *jcr) dir->fsend(_("2991 Bad accurate command\n")); return false; } - Dmsg2(2, "nb=%d msg=%s\n", nb, dir->msg); accurate_init(jcr, nb); @@ -358,7 +377,6 @@ int accurate_cmd(JCR *jcr) */ /* get current files */ while (dir->recv() >= 0) { - Dmsg1(2, "accurate_cmd fname=%s\n", dir->msg); len = strlen(dir->msg) + 1; if (len < dir->msglen) { accurate_add_file(jcr, dir->msg, dir->msg + len); @@ -369,7 +387,7 @@ int accurate_cmd(JCR *jcr) extern void *start_heap; char b1[50], b2[50], b3[50], b4[50], b5[50]; - Dmsg5(1," Heap: heap=%s smbytes=%s max_bytes=%s bufs=%s max_bufs=%s\n", + Dmsg5(dbglvl," Heap: heap=%s smbytes=%s max_bytes=%s bufs=%s max_bufs=%s\n", edit_uint64_with_commas((char *)sbrk(0)-(char *)start_heap, b1), edit_uint64_with_commas(sm_bytes, b2), edit_uint64_with_commas(sm_max_bytes, b3), @@ -380,3 +398,17 @@ int accurate_cmd(JCR *jcr) return true; } + +#ifdef USE_TCADB + +/* + * Tokyo Cabinet library doesn't use smartalloc by default + * results need to be released with real free() + */ +#undef free +void realfree(void *p) +{ + free(p); +} + +#endif