/*
Bacula® - The Network Backup Solution
- Copyright (C) 2000-2008 Free Software Foundation Europe e.V.
+ Copyright (C) 2000-2009 Free Software Foundation Europe e.V.
The main author of Bacula is Kern Sibbald, with contributions from
many others, a complete list can be found in the file AUTHORS.
#include "bacula.h"
#include "filed.h"
-static int dbglvl=200;
+static int dbglvl=100;
typedef struct PrivateCurFile {
-#ifndef USE_TCADB
hlink link;
-#endif
- char *fname; /* not stored with tchdb mode */
- time_t ctime;
- time_t mtime;
+ char *fname;
+ char *lstat;
+ char *chksum;
bool seen;
} CurFile;
-#ifdef USE_TCADB
-static void realfree(void *p); /* used by tokyo code */
+bool accurate_mark_file_as_seen(JCR *jcr, char *fname)
+{
+ if (!jcr->accurate || !jcr->file_list) {
+ return false;
+ }
+ /* TODO: just use elt->seen = 1 */
+ CurFile *temp = (CurFile *)jcr->file_list->lookup(fname);
+ if (temp) {
+ temp->seen = 1; /* records are in memory */
+ Dmsg1(dbglvl, "marked <%s> as seen\n", fname);
+ } else {
+ Dmsg1(dbglvl, "<%s> not found to be marked as seen\n", fname);
+ }
+ return true;
+}
-/*
- * Update hash element seen=1
- */
static bool accurate_mark_file_as_seen(JCR *jcr, CurFile *elt)
{
- bool ret=true;
-
- elt->seen = 1;
- if (!tcadbput(jcr->file_list,
- elt->fname, strlen(elt->fname)+1,
- elt, sizeof(CurFile)))
- { /* TODO: disabling accurate mode ? */
- Jmsg(jcr, M_ERROR, 1, _("Can't update accurate hash disk\n"));
- ret = false;
+ /* TODO: just use elt->seen = 1 */
+ CurFile *temp = (CurFile *)jcr->file_list->lookup(elt->fname);
+ if (temp) {
+ temp->seen = 1; /* records are in memory */
}
-
- return ret;
+ return true;
}
static bool accurate_lookup(JCR *jcr, char *fname, CurFile *ret)
{
bool found=false;
ret->seen = 0;
- int size;
- CurFile *elt;
- elt = (CurFile*)tcadbget(jcr->file_list,
- fname, strlen(fname)+1, &size);
- if (elt)
- {
- /* TODO: don't malloc/free results */
- found = true;
- elt->fname = fname;
- memcpy(ret, elt, sizeof(CurFile));
- realfree(elt);
-// Dmsg1(dbglvl, "lookup <%s> ok\n", fname);
+ CurFile *temp = (CurFile *)jcr->file_list->lookup(fname);
+ if (temp) {
+ memcpy(ret, temp, sizeof(CurFile));
+ found=true;
+ Dmsg1(dbglvl, "lookup <%s> ok\n", fname);
}
+
return found;
}
-/* Create tokyo dbm hash file
- * If something goes wrong, we cancel accurate mode.
- */
static bool accurate_init(JCR *jcr, int nbfile)
{
- jcr->file_list = tcadbnew();
-//
-// tchdbsetcache(jcr->file_list, 300000);
-// tchdbtune(jcr->file_list,
-// nbfile, /* nb bucket 0.5n to 4n */
-// 6, /* size of element 2^x */
-// 16,
-// 0); /* options like compression */
-//
- jcr->hash_name = get_pool_memory(PM_MESSAGE);
- POOLMEM *temp = get_pool_memory(PM_MESSAGE);
-
- if (nbfile > 500000) {
- make_unique_filename(&jcr->hash_name, jcr->JobId, "accurate");
- pm_strcat(jcr->hash_name, ".tcb");
- Mmsg(temp, "%s#bnum=%i#mode=e#opts=l",
- jcr->hash_name, nbfile*4);
- Dmsg1(dbglvl, "Doing accurate hash on disk %s\n", jcr->hash_name);
- } else {
- Dmsg0(dbglvl, "Doing accurate hash on memory\n");
- pm_strcpy(jcr->hash_name, "*");
- pm_strcpy(temp, "*");
- }
-
- if(!tcadbopen(jcr->file_list, jcr->hash_name)){
- Jmsg(jcr, M_ERROR, 1, _("Can't open accurate hash disk\n"));
- Jmsg(jcr, M_INFO, 1, _("Disabling accurate mode\n"));
- tcadbdel(jcr->file_list);
- jcr->file_list = NULL;
- jcr->accurate = false;
- }
- free_pool_memory(temp);
- return jcr->file_list != NULL;
+ CurFile *elt = NULL;
+ jcr->file_list = (htable *)malloc(sizeof(htable));
+ jcr->file_list->init(elt, &elt->link, nbfile);
+ return true;
}
-/* This function is called at the end of backup
- * We walk over all hash disk element, and we check
- * for elt.seen.
- */
-bool accurate_send_deleted_list(JCR *jcr)
+static bool accurate_send_base_file_list(JCR *jcr)
{
- char *key;
CurFile *elt;
- int size;
+ struct stat statc;
+ int32_t LinkFIc;
FF_PKT *ff_pkt;
int stream = STREAM_UNIX_ATTRIBUTES;
- if (!jcr->accurate || jcr->get_JobLevel() == L_FULL) {
- goto bail_out;
+ if (!jcr->accurate || jcr->getJobLevel() != L_FULL) {
+ return true;
}
if (jcr->file_list == NULL) {
- goto bail_out;
+ return true;
}
ff_pkt = init_find_files();
- ff_pkt->type = FT_DELETED;
+ ff_pkt->type = FT_BASE;
- /* traverse records */
- tcadbiterinit(jcr->file_list);
- while((key = tcadbiternext2(jcr->file_list)) != NULL) {
- elt = (CurFile *) tcadbget(jcr->file_list,
- key, strlen(key)+1, &size);
- if (elt)
- {
- if (!elt->seen) { /* already seen */
- ff_pkt->fname = key;
- ff_pkt->statp.st_mtime = elt->mtime;
- ff_pkt->statp.st_ctime = elt->ctime;
- encode_and_send_attributes(jcr, ff_pkt, stream);
- }
- realfree(elt);
+ foreach_htable(elt, jcr->file_list) {
+ if (elt->seen) {
+ Dmsg2(dbglvl, "base file fname=%s seen=%i\n", elt->fname, elt->seen);
+ /* TODO: skip the decode and use directly the lstat field */
+ decode_stat(elt->lstat, &statc, &LinkFIc); /* decode catalog stat */
+ ff_pkt->fname = elt->fname;
+ ff_pkt->statp = statc;
+ encode_and_send_attributes(jcr, ff_pkt, stream);
+// free(elt->fname);
}
- realfree(key); /* tokyo cabinet have to use real free() */
}
term_find_files(ff_pkt);
-bail_out:
- /* TODO: clean htable when this function is not reached ? */
- if (jcr->file_list) {
- if(!tcadbclose(jcr->file_list)){
- Jmsg(jcr, M_ERROR, 1, _("Can't close accurate hash disk\n"));
- }
-
- /* delete the object */
- tcadbdel(jcr->file_list);
- if (!bstrcmp(jcr->hash_name, "*")) {
- unlink(jcr->hash_name);
- }
-
- free_pool_memory(jcr->hash_name);
- jcr->hash_name = NULL;
- jcr->file_list = NULL;
- }
- return true;
-}
-
-#else /* HTABLE mode */
-
-static bool accurate_mark_file_as_seen(JCR *jcr, CurFile *elt)
-{
- CurFile *temp = (CurFile *)jcr->file_list->lookup(elt->fname);
- temp->seen = 1; /* records are in memory */
return true;
}
-static bool accurate_lookup(JCR *jcr, char *fname, CurFile *ret)
-{
- bool found=false;
- ret->seen = 0;
-
- CurFile *temp = (CurFile *)jcr->file_list->lookup(fname);
- if (temp) {
- memcpy(ret, temp, sizeof(CurFile));
- found=true;
-// Dmsg1(dbglvl, "lookup <%s> ok\n", fname);
- }
-
- return found;
-}
-
-static bool accurate_init(JCR *jcr, int nbfile)
-{
- CurFile *elt = NULL;
- jcr->file_list = (htable *)malloc(sizeof(htable));
- jcr->file_list->init(elt, &elt->link, nbfile);
- return true;
-}
/* This function is called at the end of backup
* We walk over all hash disk element, and we check
* for elt.seen.
*/
-bool accurate_send_deleted_list(JCR *jcr)
+static bool accurate_send_deleted_list(JCR *jcr)
{
CurFile *elt;
+ struct stat statc;
+ int32_t LinkFIc;
FF_PKT *ff_pkt;
int stream = STREAM_UNIX_ATTRIBUTES;
- if (!jcr->accurate || jcr->get_JobLevel() == L_FULL) {
- goto bail_out;
+ if (!jcr->accurate) {
+ return true;
}
if (jcr->file_list == NULL) {
- goto bail_out;
+ return true;
}
ff_pkt = init_find_files();
ff_pkt->type = FT_DELETED;
foreach_htable(elt, jcr->file_list) {
- if (!elt->seen) { /* already seen */
- Dmsg2(dbglvl, "deleted fname=%s seen=%i\n", elt->fname, elt->seen);
- ff_pkt->fname = elt->fname;
- ff_pkt->statp.st_mtime = elt->mtime;
- ff_pkt->statp.st_ctime = elt->ctime;
- encode_and_send_attributes(jcr, ff_pkt, stream);
+ if (elt->seen || plugin_check_file(jcr, elt->fname)) {
+ continue;
}
-// free(elt->fname);
+ Dmsg2(dbglvl, "deleted fname=%s seen=%i\n", elt->fname, elt->seen);
+ /* TODO: skip the decode and use directly the lstat field */
+ decode_stat(elt->lstat, &statc, &LinkFIc); /* decode catalog stat */
+ ff_pkt->fname = elt->fname;
+ ff_pkt->statp.st_mtime = statc.st_mtime;
+ ff_pkt->statp.st_ctime = statc.st_ctime;
+ encode_and_send_attributes(jcr, ff_pkt, stream);
+// free(elt->fname);
}
term_find_files(ff_pkt);
-bail_out:
- /* TODO: clean htable when this function is not reached ? */
+ return true;
+}
+
+void accurate_free(JCR *jcr)
+{
if (jcr->file_list) {
jcr->file_list->destroy();
free(jcr->file_list);
jcr->file_list = NULL;
}
- return true;
}
-#endif /* common code */
+/* Send the deleted or the base file list and cleanup */
+bool accurate_finish(JCR *jcr)
+{
+ bool ret=true;
+ if (jcr->accurate) {
+ if (jcr->getJobLevel() == L_FULL) {
+ ret = accurate_send_base_file_list(jcr);
+ } else {
+ ret = accurate_send_deleted_list(jcr);
+ }
+
+ accurate_free(jcr);
+ if (jcr->getJobLevel() == L_FULL) {
+ Jmsg(jcr, M_INFO, 0, _("Space saved with Base jobs: %lld MB\n"),
+ jcr->base_size/(1024*1024));
+ }
+ }
+ return ret;
+}
-static bool accurate_add_file(JCR *jcr, char *fname, char *lstat)
+static bool accurate_add_file(JCR *jcr, uint32_t len,
+ char *fname, char *lstat, char *chksum)
{
bool ret = true;
CurFile elt;
- struct stat statp;
- int LinkFIc;
- decode_stat(lstat, &statp, &LinkFIc); /* decode catalog stat */
- elt.ctime = statp.st_ctime;
- elt.mtime = statp.st_mtime;
elt.seen = 0;
-#ifdef USE_TCADB
- if (!tcadbput(jcr->file_list,
- fname, strlen(fname)+1,
- &elt, sizeof(CurFile)))
- {
- Jmsg(jcr, M_ERROR, 1, _("Can't update accurate hash disk ERR=%s\n"));
- ret = false;
- }
-#else /* HTABLE */
CurFile *item;
/* we store CurFile, fname and ctime/mtime in the same chunk */
- item = (CurFile *)jcr->file_list->hash_malloc(sizeof(CurFile)+strlen(fname)+1);
+ item = (CurFile *)jcr->file_list->hash_malloc(sizeof(CurFile)+len+3);
memcpy(item, &elt, sizeof(CurFile));
+
item->fname = (char *)item+sizeof(CurFile);
strcpy(item->fname, fname);
+
+ item->lstat = item->fname+strlen(item->fname)+1;
+ strcpy(item->lstat, lstat);
+
+ item->chksum = item->lstat+strlen(item->lstat)+1;
+ strcpy(item->chksum, chksum);
+
jcr->file_list->insert(item->fname, item);
-#endif
-// Dmsg2(dbglvl, "add fname=<%s> lstat=%s\n", fname, lstat);
+ Dmsg3(dbglvl, "add fname=<%s> lstat=%s chksum=%s\n", fname, lstat, chksum);
return ret;
}
* We check in file_list hash if fname have been backuped
* the last time. After we can compare Lstat field.
* Full Lstat usage have been removed on 6612
+ *
+ * Returns: true if file has changed (must be backed up)
+ * false file not changed
*/
bool accurate_check_file(JCR *jcr, FF_PKT *ff_pkt)
{
+ int digest_stream = STREAM_NONE;
+ DIGEST *digest = NULL;
+
+ struct stat statc;
+ int32_t LinkFIc;
bool stat = false;
+ char *opts;
char *fname;
CurFile elt;
- if (!jcr->accurate || jcr->get_JobLevel() == L_FULL) {
+ if (!jcr->accurate) {
return true;
}
goto bail_out;
}
- if (elt.mtime != ff_pkt->statp.st_mtime) {
-// Jmsg(jcr, M_SAVED, 0, _("%s st_mtime differs\n"), fname);
- Dmsg1(dbglvl, "%s st_mtime differs\n", fname);
- stat = true;
- } else if (elt.ctime != ff_pkt->statp.st_ctime) {
-// Jmsg(jcr, M_SAVED, 0, _("%s st_ctime differs\n"), fname);
- Dmsg1(dbglvl, "%s st_ctime differs\n", fname);
- stat = true;
+ decode_stat(elt.lstat, &statc, &LinkFIc); /* decode catalog stat */
+
+ if (jcr->getJobLevel() == L_FULL) {
+ opts = ff_pkt->BaseJobOpts;
+ } else {
+ opts = ff_pkt->AccurateOpts;
+ }
+
+ /*
+ * Loop over options supplied by user and verify the
+ * fields he requests.
+ */
+ for (char *p=opts; !stat && *p; p++) {
+ char ed1[30], ed2[30];
+ switch (*p) {
+ case 'i': /* compare INODEs */
+ if (statc.st_ino != ff_pkt->statp.st_ino) {
+ Dmsg3(dbglvl-1, "%s st_ino differ. Cat: %s File: %s\n",
+ fname,
+ edit_uint64((uint64_t)statc.st_ino, ed1),
+ edit_uint64((uint64_t)ff_pkt->statp.st_ino, ed2));
+ stat = true;
+ }
+ break;
+ case 'p': /* permissions bits */
+ /* TODO: If something change only in perm, user, group
+ * Backup only the attribute stream
+ */
+ if (statc.st_mode != ff_pkt->statp.st_mode) {
+ Dmsg3(dbglvl-1, "%s st_mode differ. Cat: %x File: %x\n",
+ fname,
+ (uint32_t)statc.st_mode, (uint32_t)ff_pkt->statp.st_mode);
+ stat = true;
+ }
+ break;
+ case 'n': /* number of links */
+ if (statc.st_nlink != ff_pkt->statp.st_nlink) {
+ Dmsg3(dbglvl-1, "%s st_nlink differ. Cat: %d File: %d\n",
+ fname,
+ (uint32_t)statc.st_nlink, (uint32_t)ff_pkt->statp.st_nlink);
+ stat = true;
+ }
+ break;
+ case 'u': /* user id */
+ if (statc.st_uid != ff_pkt->statp.st_uid) {
+ Dmsg3(dbglvl-1, "%s st_uid differ. Cat: %u File: %u\n",
+ fname,
+ (uint32_t)statc.st_uid, (uint32_t)ff_pkt->statp.st_uid);
+ stat = true;
+ }
+ break;
+ case 'g': /* group id */
+ if (statc.st_gid != ff_pkt->statp.st_gid) {
+ Dmsg3(dbglvl-1, "%s st_gid differ. Cat: %u File: %u\n",
+ fname,
+ (uint32_t)statc.st_gid, (uint32_t)ff_pkt->statp.st_gid);
+ stat = true;
+ }
+ break;
+ case 's': /* size */
+ if (statc.st_size != ff_pkt->statp.st_size) {
+ Dmsg3(dbglvl-1, "%s st_size differ. Cat: %s File: %s\n",
+ fname,
+ edit_uint64((uint64_t)statc.st_size, ed1),
+ edit_uint64((uint64_t)ff_pkt->statp.st_size, ed2));
+ stat = true;
+ }
+ break;
+ case 'a': /* access time */
+ if (statc.st_atime != ff_pkt->statp.st_atime) {
+ Dmsg1(dbglvl-1, "%s st_atime differs\n", fname);
+ stat = true;
+ }
+ break;
+ case 'm': /* modification time */
+ if (statc.st_mtime != ff_pkt->statp.st_mtime) {
+ Dmsg1(dbglvl-1, "%s st_mtime differs\n", fname);
+ stat = true;
+ }
+ break;
+ case 'c': /* ctime */
+ if (statc.st_ctime != ff_pkt->statp.st_ctime) {
+ Dmsg1(dbglvl-1, "%s st_ctime differs\n", fname);
+ stat = true;
+ }
+ break;
+ case 'd': /* file size decrease */
+ if (statc.st_size > ff_pkt->statp.st_size) {
+ Dmsg3(dbglvl-1, "%s st_size decrease. Cat: %s File: %s\n",
+ fname,
+ edit_uint64((uint64_t)statc.st_size, ed1),
+ edit_uint64((uint64_t)ff_pkt->statp.st_size, ed2));
+ stat = true;
+ }
+ break;
+
+ /* TODO: cleanup and factorise this function with verify.c */
+ case '5': /* compare MD5 */
+ case '1': /* compare SHA1 */
+ /*
+ * The remainder of the function is all about getting the checksum.
+ * First we initialise, then we read files, other streams and Finder Info.
+ */
+ if (!stat && ff_pkt->type != FT_LNKSAVED &&
+ (S_ISREG(ff_pkt->statp.st_mode) &&
+ ff_pkt->flags & (FO_MD5|FO_SHA1|FO_SHA256|FO_SHA512)))
+ {
+
+ if (!*elt.chksum) {
+ Jmsg(jcr, M_WARNING, 0, _("Can't verify checksum for %s\n"),
+ ff_pkt->fname);
+ stat = true;
+ break;
+ }
+
+ /*
+ * Create our digest context. If this fails, the digest will be set
+ * to NULL and not used.
+ */
+ if (ff_pkt->flags & FO_MD5) {
+ digest = crypto_digest_new(jcr, CRYPTO_DIGEST_MD5);
+ digest_stream = STREAM_MD5_DIGEST;
+
+ } else if (ff_pkt->flags & FO_SHA1) {
+ digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA1);
+ digest_stream = STREAM_SHA1_DIGEST;
+
+ } else if (ff_pkt->flags & FO_SHA256) {
+ digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA256);
+ digest_stream = STREAM_SHA256_DIGEST;
+
+ } else if (ff_pkt->flags & FO_SHA512) {
+ digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA512);
+ digest_stream = STREAM_SHA512_DIGEST;
+ }
+
+ /* Did digest initialization fail? */
+ if (digest_stream != STREAM_NONE && digest == NULL) {
+ Jmsg(jcr, M_WARNING, 0, _("%s digest initialization failed\n"),
+ stream_to_ascii(digest_stream));
+ }
+
+ /* compute MD5 or SHA1 hash */
+ if (digest) {
+ char md[CRYPTO_DIGEST_MAX_SIZE];
+ uint32_t size;
+
+ size = sizeof(md);
+
+ if (digest_file(jcr, ff_pkt, digest) != 0) {
+ jcr->JobErrors++;
+
+ } else if (crypto_digest_finalize(digest, (uint8_t *)md, &size)) {
+ char *digest_buf;
+ const char *digest_name;
+
+ digest_buf = (char *)malloc(BASE64_SIZE(size));
+ digest_name = crypto_digest_name(digest);
+
+ bin_to_base64(digest_buf, BASE64_SIZE(size), md, size, true);
+
+ if (strcmp(digest_buf, elt.chksum)) {
+ Dmsg3(dbglvl-1, "%s chksum diff. Cat: %s File: %s\n",
+ fname,
+ elt.chksum,
+ digest_buf);
+ stat = true;
+ }
+
+ free(digest_buf);
+ }
+ crypto_digest_free(digest);
+ }
+ }
+
+ break;
+ case ':':
+ case 'J':
+ case 'C':
+ default:
+ break;
+ }
}
- accurate_mark_file_as_seen(jcr, &elt);
- Dmsg2(dbglvl, "accurate %s = %i\n", fname, stat);
+ /* In Incr/Diff accurate mode, we mark all files as seen
+ * When in Full+Base mode, we mark only if the file match exactly
+ */
+ if (jcr->getJobLevel() == L_FULL) {
+ if (!stat) {
+ /* compute space saved with basefile */
+ jcr->base_size += ff_pkt->statp.st_size;
+ accurate_mark_file_as_seen(jcr, &elt);
+ }
+ } else {
+ accurate_mark_file_as_seen(jcr, &elt);
+ }
bail_out:
unstrip_path(ff_pkt);
int accurate_cmd(JCR *jcr)
{
BSOCK *dir = jcr->dir_bsock;
- int len;
+ int lstat_pos, chksum_pos;
int32_t nb;
- if (!jcr->accurate || job_canceled(jcr) || jcr->get_JobLevel()==L_FULL) {
+ if (job_canceled(jcr)) {
return true;
}
-
if (sscanf(dir->msg, "accurate files=%ld", &nb) != 1) {
dir->fsend(_("2991 Bad accurate command\n"));
return false;
}
+ jcr->accurate = true;
+
accurate_init(jcr, nb);
/*
* buffer = sizeof(CurFile) + dirmsg
- * dirmsg = fname + \0 + lstat
+ * dirmsg = fname + \0 + lstat + \0 + checksum + \0
*/
/* get current files */
while (dir->recv() >= 0) {
- len = strlen(dir->msg) + 1;
- if (len < dir->msglen) {
- accurate_add_file(jcr, dir->msg, dir->msg + len);
+ lstat_pos = strlen(dir->msg) + 1;
+ if (lstat_pos < dir->msglen) {
+ chksum_pos = lstat_pos + strlen(dir->msg + lstat_pos) + 1;
+
+ if (chksum_pos >= dir->msglen) {
+ chksum_pos = lstat_pos - 1; /* tweak: no checksum, point to the last \0 */
+ }
+
+ accurate_add_file(jcr, dir->msglen,
+ dir->msg, /* Path */
+ dir->msg + lstat_pos, /* LStat */
+ dir->msg + chksum_pos); /* CheckSum */
}
}
return true;
}
-
-#ifdef USE_TCADB
-
-/*
- * Tokyo Cabinet library doesn't use smartalloc by default
- * results need to be released with real free()
- */
-#undef free
-void realfree(void *p)
-{
- free(p);
-}
-
-#endif
projects / bacula / bacula / blobdiff