#include "bacula.h"
#include "filed.h"
+static int dbglvl=200;
+
typedef struct PrivateCurFile {
-#ifndef USE_TCHDB
+#ifndef USE_TCADB
hlink link;
#endif
char *fname; /* not stored with tchdb mode */
bool seen;
} CurFile;
-bool accurate_add_file(JCR *jcr, char *fname, char *lstat)
-{
- CurFile elt;
- struct stat statp;
- int LinkFIc;
- decode_stat(lstat, &statp, &LinkFIc); /* decode catalog stat */
- elt.ctime = statp.st_ctime;
- elt.mtime = statp.st_mtime;
- elt.seen = 0;
-
-#ifdef USE_TCHDB
- if (!tchdbputasync(jcr->file_list,
- fname, strlen(fname)+1,
- &elt, sizeof(CurFile)))
- {
- Dmsg1(2, "Can't add <%s> to file_list\n", fname);
- /* TODO: check error */
- }
-#else /* HTABLE */
- CurFile *item;
- /* we store CurFile, fname and ctime/mtime in the same chunk */
- item = (CurFile *)jcr->file_list->hash_malloc(sizeof(CurFile)+strlen(fname)+1);
- memcpy(item, &elt, sizeof(CurFile));
- item->fname = (char *)item+sizeof(CurFile);
- strcpy(item->fname, fname);
- jcr->file_list->insert(item->fname, item);
-#endif
-
- Dmsg2(2, "add fname=<%s> lstat=%s\n", fname, lstat);
- return true;
-}
+#ifdef USE_TCADB
+static void realfree(void *p); /* used by tokyo code */
-bool accurate_mark_file_as_seen(JCR *jcr, CurFile *elt)
+/*
+ * Update hash element seen=1
+ */
+static bool accurate_mark_file_as_seen(JCR *jcr, CurFile *elt)
{
bool ret=true;
- CurFile deb;
-#ifdef USE_TCHDB
elt->seen = 1;
- if (!tchdbputasync(jcr->file_list,
- elt->fname, strlen(elt->fname)+1,
- elt, sizeof(CurFile)))
- {
- Dmsg1(2, "can't update <%s>\n", elt->fname);
- ret = false; /* TODO: add error message */
+ if (!tcadbput(jcr->file_list,
+ elt->fname, strlen(elt->fname)+1,
+ elt, sizeof(CurFile)))
+ { /* TODO: disabling accurate mode ? */
+ Jmsg(jcr, M_ERROR, 1, _("Can't update accurate hash disk\n"));
+ ret = false;
}
-#else /* HTABLE */
- CurFile *temp = (CurFile *)jcr->file_list->lookup(elt->fname);
- temp->seen = 1;
-#endif
return ret;
}
-bool accurate_lookup(JCR *jcr, char *fname, CurFile *ret)
+static bool accurate_lookup(JCR *jcr, char *fname, CurFile *ret)
{
bool found=false;
ret->seen = 0;
+ int size;
+ CurFile *elt;
-#ifdef USE_TCHDB
- if (tchdbget3(jcr->file_list,
- fname, strlen(fname)+1,
- ret, sizeof(CurFile)) != -1)
+ elt = (CurFile*)tcadbget(jcr->file_list,
+ fname, strlen(fname)+1, &size);
+ if (elt)
{
+ /* TODO: don't malloc/free results */
found = true;
- ret->fname = fname;
+ elt->fname = fname;
+ memcpy(ret, elt, sizeof(CurFile));
+ realfree(elt);
+// Dmsg1(dbglvl, "lookup <%s> ok\n", fname);
}
+ return found;
+}
+
+/* Create tokyo dbm hash file
+ * If something goes wrong, we cancel accurate mode.
+ */
+static bool accurate_init(JCR *jcr, int nbfile)
+{
+ jcr->file_list = tcadbnew();
+//
+// tchdbsetcache(jcr->file_list, 300000);
+// tchdbtune(jcr->file_list,
+// nbfile, /* nb bucket 0.5n to 4n */
+// 6, /* size of element 2^x */
+// 16,
+// 0); /* options like compression */
+//
+ jcr->hash_name = get_pool_memory(PM_MESSAGE);
+ POOLMEM *temp = get_pool_memory(PM_MESSAGE);
+
+ if (nbfile > 500000) {
+ make_unique_filename(&jcr->hash_name, jcr->JobId, "accurate");
+ pm_strcat(jcr->hash_name, ".tcb");
+ Mmsg(temp, "%s#bnum=%i#mode=e#opts=l",
+ jcr->hash_name, nbfile*4);
+ Dmsg1(dbglvl, "Doing accurate hash on disk %s\n", jcr->hash_name);
+ } else {
+ Dmsg0(dbglvl, "Doing accurate hash on memory\n");
+ pm_strcpy(jcr->hash_name, "*");
+ pm_strcpy(temp, "*");
+ }
+
+ if(!tcadbopen(jcr->file_list, jcr->hash_name)){
+ Jmsg(jcr, M_ERROR, 1, _("Can't open accurate hash disk\n"));
+ Jmsg(jcr, M_INFO, 1, _("Disabling accurate mode\n"));
+ tcadbdel(jcr->file_list);
+ jcr->file_list = NULL;
+ jcr->accurate = false;
+ }
+ free_pool_memory(temp);
+ return jcr->file_list != NULL;
+}
+
+/* This function is called at the end of backup
+ * We walk over all hash disk element, and we check
+ * for elt.seen.
+ */
+bool accurate_send_deleted_list(JCR *jcr)
+{
+ char *key;
+ CurFile *elt;
+ int size;
+ FF_PKT *ff_pkt;
+ int stream = STREAM_UNIX_ATTRIBUTES;
+
+ if (!jcr->accurate || jcr->JobLevel == L_FULL) {
+ goto bail_out;
+ }
+
+ if (jcr->file_list == NULL) {
+ goto bail_out;
+ }
+
+ ff_pkt = init_find_files();
+ ff_pkt->type = FT_DELETED;
+
+ /* traverse records */
+ tcadbiterinit(jcr->file_list);
+ while((key = tcadbiternext2(jcr->file_list)) != NULL) {
+ elt = (CurFile *) tcadbget(jcr->file_list,
+ key, strlen(key)+1, &size);
+ if (elt)
+ {
+ if (!elt->seen) { /* already seen */
+ ff_pkt->fname = key;
+ ff_pkt->statp.st_mtime = elt->mtime;
+ ff_pkt->statp.st_ctime = elt->ctime;
+ encode_and_send_attributes(jcr, ff_pkt, stream);
+ }
+ realfree(elt);
+ }
+ realfree(key); /* tokyo cabinet have to use real free() */
+ }
+
+ term_find_files(ff_pkt);
+bail_out:
+ /* TODO: clean htable when this function is not reached ? */
+ if (jcr->file_list) {
+ if(!tcadbclose(jcr->file_list)){
+ Jmsg(jcr, M_ERROR, 1, _("Can't close accurate hash disk\n"));
+ }
+
+ /* delete the object */
+ tcadbdel(jcr->file_list);
+ if (!bstrcmp(jcr->hash_name, "*")) {
+ unlink(jcr->hash_name);
+ }
+
+ free_pool_memory(jcr->hash_name);
+ jcr->hash_name = NULL;
+ jcr->file_list = NULL;
+ }
+ return true;
+}
+
+#else /* HTABLE mode */
+
+static bool accurate_mark_file_as_seen(JCR *jcr, CurFile *elt)
+{
+ CurFile *temp = (CurFile *)jcr->file_list->lookup(elt->fname);
+ temp->seen = 1; /* records are in memory */
+ return true;
+}
+
+static bool accurate_lookup(JCR *jcr, char *fname, CurFile *ret)
+{
+ bool found=false;
+ ret->seen = 0;
-#else /* HTABLE */
CurFile *temp = (CurFile *)jcr->file_list->lookup(fname);
if (temp) {
memcpy(ret, temp, sizeof(CurFile));
found=true;
- }
-#endif
-
- if (found) {
- Dmsg1(2, "lookup <%s> ok\n", fname);
+// Dmsg1(dbglvl, "lookup <%s> ok\n", fname);
}
return found;
}
-bool accurate_init(JCR *jcr, int nbfile)
+static bool accurate_init(JCR *jcr, int nbfile)
{
-#ifdef USE_TCHDB
- jcr->file_list = tchdbnew();
- tchdbsetcache(jcr->file_list, 300000);
- tchdbtune(jcr->file_list,
- nbfile, /* nb bucket 0.5n to 4n */
- 7, /* size of element 2^x */
- 16,
- 0); /* options like compression */
- /* TODO: make accurate file unique */
- POOL_MEM buf;
- Mmsg(buf, "/tmp/casket.hdb.%i", jcr->JobId);
- if(!tchdbopen(jcr->file_list, buf.c_str(), HDBOWRITER | HDBOCREAT)){
- /* TODO: handle error creation */
- //ecode = tchdbecode(hdb);
- //fprintf(stderr, "open error: %s\n", tchdberrmsg(ecode));
- }
-
-#else /* HTABLE */
CurFile *elt=NULL;
jcr->file_list = (htable *)malloc(sizeof(htable));
jcr->file_list->init(elt, &elt->link, nbfile);
-#endif
+ return true;
+}
+
+/* This function is called at the end of backup
+ * We walk over all hash disk element, and we check
+ * for elt.seen.
+ */
+bool accurate_send_deleted_list(JCR *jcr)
+{
+ CurFile *elt;
+ FF_PKT *ff_pkt;
+ int stream = STREAM_UNIX_ATTRIBUTES;
+ if (!jcr->accurate || jcr->JobLevel == L_FULL) {
+ goto bail_out;
+ }
+
+ if (jcr->file_list == NULL) {
+ goto bail_out;
+ }
+
+ ff_pkt = init_find_files();
+ ff_pkt->type = FT_DELETED;
+
+ foreach_htable (elt, jcr->file_list) {
+ if (!elt->seen) { /* already seen */
+ Dmsg2(dbglvl, "deleted fname=%s seen=%i\n", elt->fname, elt->seen);
+ ff_pkt->fname = elt->fname;
+ ff_pkt->statp.st_mtime = elt->mtime;
+ ff_pkt->statp.st_ctime = elt->ctime;
+ encode_and_send_attributes(jcr, ff_pkt, stream);
+ }
+// free(elt->fname);
+ }
+
+ term_find_files(ff_pkt);
+bail_out:
+ /* TODO: clean htable when this function is not reached ? */
+ if (jcr->file_list) {
+ jcr->file_list->destroy();
+ free(jcr->file_list);
+ jcr->file_list = NULL;
+ }
return true;
}
+#endif /* common code */
+
+static bool accurate_add_file(JCR *jcr, char *fname, char *lstat)
+{
+ bool ret = true;
+ CurFile elt;
+ struct stat statp;
+ int LinkFIc;
+ decode_stat(lstat, &statp, &LinkFIc); /* decode catalog stat */
+ elt.ctime = statp.st_ctime;
+ elt.mtime = statp.st_mtime;
+ elt.seen = 0;
+
+#ifdef USE_TCADB
+ if (!tcadbput(jcr->file_list,
+ fname, strlen(fname)+1,
+ &elt, sizeof(CurFile)))
+ {
+ Jmsg(jcr, M_ERROR, 1, _("Can't update accurate hash disk ERR=%s\n"));
+ ret = false;
+ }
+#else /* HTABLE */
+ CurFile *item;
+ /* we store CurFile, fname and ctime/mtime in the same chunk */
+ item = (CurFile *)jcr->file_list->hash_malloc(sizeof(CurFile)+strlen(fname)+1);
+ memcpy(item, &elt, sizeof(CurFile));
+ item->fname = (char *)item+sizeof(CurFile);
+ strcpy(item->fname, fname);
+ jcr->file_list->insert(item->fname, item);
+#endif
+
+// Dmsg2(dbglvl, "add fname=<%s> lstat=%s\n", fname, lstat);
+ return ret;
+}
+
/*
* This function is called for each file seen in fileset.
* We check in file_list hash if fname have been backuped
}
if (!accurate_lookup(jcr, fname, &elt)) {
- Dmsg1(2, "accurate %s (not found)\n", fname);
+ Dmsg1(dbglvl, "accurate %s (not found)\n", fname);
stat = true;
goto bail_out;
}
if (elt.seen) { /* file has been seen ? */
- Dmsg1(2, "accurate %s (already seen)\n", fname);
+ Dmsg1(dbglvl, "accurate %s (already seen)\n", fname);
goto bail_out;
}
stat = true;
}
- if (stat) {
- Dmsg4(1, "%i = %i\t%i = %i\n", elt.mtime, ff_pkt->statp.st_mtime,
- elt.ctime, ff_pkt->statp.st_ctime);
- }
-
accurate_mark_file_as_seen(jcr, &elt);
- Dmsg2(2, "accurate %s = %i\n", fname, stat);
+ Dmsg2(dbglvl, "accurate %s = %i\n", fname, stat);
bail_out:
unstrip_path(ff_pkt);
dir->fsend(_("2991 Bad accurate command\n"));
return false;
}
- Dmsg2(2, "nb=%d msg=%s\n", nb, dir->msg);
accurate_init(jcr, nb);
*/
/* get current files */
while (dir->recv() >= 0) {
- Dmsg1(2, "accurate_cmd fname=%s\n", dir->msg);
len = strlen(dir->msg) + 1;
if (len < dir->msglen) {
accurate_add_file(jcr, dir->msg, dir->msg + len);
extern void *start_heap;
char b1[50], b2[50], b3[50], b4[50], b5[50];
- Dmsg5(1," Heap: heap=%s smbytes=%s max_bytes=%s bufs=%s max_bufs=%s\n",
+ Dmsg5(dbglvl," Heap: heap=%s smbytes=%s max_bytes=%s bufs=%s max_bufs=%s\n",
edit_uint64_with_commas((char *)sbrk(0)-(char *)start_heap, b1),
edit_uint64_with_commas(sm_bytes, b2),
edit_uint64_with_commas(sm_max_bytes, b3),
return true;
}
-bool accurate_send_deleted_list(JCR *jcr)
-{
- CurFile *elt;
- FF_PKT *ff_pkt;
- int stream = STREAM_UNIX_ATTRIBUTES;
-
- if (!jcr->accurate || jcr->JobLevel == L_FULL) {
- goto bail_out;
- }
-
- if (jcr->file_list == NULL) {
- goto bail_out;
- }
-
- ff_pkt = init_find_files();
- ff_pkt->type = FT_DELETED;
-
-#ifdef USE_TCHDB
- char *key;
- CurFile item;
- elt = &item;
- /* traverse records */
- tchdbiterinit(jcr->file_list);
- while((key = tchdbiternext2(jcr->file_list)) != NULL) {
- if (tchdbget3(jcr->file_list,
- key, strlen(key)+1,
- elt, sizeof(CurFile)) != -1)
- {
- if (!elt->seen) {
- ff_pkt->fname = key;
- ff_pkt->statp.st_mtime = elt->mtime;
- ff_pkt->statp.st_ctime = elt->ctime;
- encode_and_send_attributes(jcr, ff_pkt, stream);
- Dmsg1(2, "deleted <%s>\n", key);
- }
- // free(key);
-
- } else { /* TODO: add error message */
- Dmsg1(2, "No value for <%s> key\n", key);
- }
- }
-#else
- foreach_htable (elt, jcr->file_list) {
- if (!elt->seen) { /* already seen */
- Dmsg2(1, "deleted fname=%s seen=%i\n", elt->fname, elt->seen);
- ff_pkt->fname = elt->fname;
- ff_pkt->statp.st_mtime = elt->mtime;
- ff_pkt->statp.st_ctime = elt->ctime;
- encode_and_send_attributes(jcr, ff_pkt, stream);
- }
-// free(elt->fname);
- }
-#endif
+#ifdef USE_TCADB
- term_find_files(ff_pkt);
-bail_out:
- /* TODO: clean htable when this function is not reached ? */
- if (jcr->file_list) {
-#ifdef USE_TCHDB
- if(!tchdbclose(jcr->file_list)){
-// ecode = tchdbecode(hdb);
-// fprintf(stderr, "close error: %s\n", tchdberrmsg(ecode));
- }
+/*
+ * Tokyo Cabinet library doesn't use smartalloc by default
+ * results need to be released with real free()
+ */
+#undef free
+void realfree(void *p)
+{
+ free(p);
+}
- /* delete the object */
- tchdbdel(jcr->file_list);
- POOL_MEM buf;
- Mmsg(buf, "/tmp/casket.hdb.%i", jcr->JobId);
-// unlink("/tmp/casket.hdb");
-#else
- jcr->file_list->destroy();
- free(jcr->file_list);
#endif
- jcr->file_list = NULL;
- }
- return true;
-}
projects / bacula / bacula / blobdiff