2 Bacula® - The Network Backup Solution
4 Copyright (C) 2000-2009 Free Software Foundation Europe e.V.
6 The main author of Bacula is Kern Sibbald, with contributions from
7 many others, a complete list can be found in the file AUTHORS.
8 This program is Free Software; you can redistribute it and/or
9 modify it under the terms of version three of the GNU Affero General Public
10 License as published by the Free Software Foundation and included
13 This program is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 General Public License for more details.
18 You should have received a copy of the GNU Affero General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
23 Bacula® is a registered trademark of Kern Sibbald.
24 The licensor of Bacula is the Free Software Foundation Europe
25 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
26 Switzerland, email:ftf@fsfeurope.org.
36 static int dbglvl=100;
38 typedef struct PrivateCurFile {
47 bool accurate_mark_file_as_seen(JCR *jcr, char *fname)
49 if (!jcr->accurate || !jcr->file_list) {
52 /* TODO: just use elt->seen = 1 */
53 CurFile *temp = (CurFile *)jcr->file_list->lookup(fname);
55 temp->seen = 1; /* records are in memory */
56 Dmsg1(dbglvl, "marked <%s> as seen\n", fname);
58 Dmsg1(dbglvl, "<%s> not found to be marked as seen\n", fname);
63 static bool accurate_mark_file_as_seen(JCR *jcr, CurFile *elt)
65 /* TODO: just use elt->seen = 1 */
66 CurFile *temp = (CurFile *)jcr->file_list->lookup(elt->fname);
68 temp->seen = 1; /* records are in memory */
73 static bool accurate_lookup(JCR *jcr, char *fname, CurFile *ret)
78 CurFile *temp = (CurFile *)jcr->file_list->lookup(fname);
80 memcpy(ret, temp, sizeof(CurFile));
82 Dmsg1(dbglvl, "lookup <%s> ok\n", fname);
88 static bool accurate_init(JCR *jcr, int nbfile)
91 jcr->file_list = (htable *)malloc(sizeof(htable));
92 jcr->file_list->init(elt, &elt->link, nbfile);
96 static bool accurate_send_base_file_list(JCR *jcr)
102 int stream = STREAM_UNIX_ATTRIBUTES;
104 if (!jcr->accurate || jcr->getJobLevel() != L_FULL) {
108 if (jcr->file_list == NULL) {
112 ff_pkt = init_find_files();
113 ff_pkt->type = FT_BASE;
115 foreach_htable(elt, jcr->file_list) {
117 Dmsg2(dbglvl, "base file fname=%s seen=%i\n", elt->fname, elt->seen);
118 /* TODO: skip the decode and use directly the lstat field */
119 decode_stat(elt->lstat, &statc, &LinkFIc); /* decode catalog stat */
120 ff_pkt->fname = elt->fname;
121 ff_pkt->statp = statc;
122 encode_and_send_attributes(jcr, ff_pkt, stream);
127 term_find_files(ff_pkt);
132 /* This function is called at the end of backup
133 * We walk over all hash disk element, and we check
136 static bool accurate_send_deleted_list(JCR *jcr)
142 int stream = STREAM_UNIX_ATTRIBUTES;
144 if (!jcr->accurate) {
148 if (jcr->file_list == NULL) {
152 ff_pkt = init_find_files();
153 ff_pkt->type = FT_DELETED;
155 foreach_htable(elt, jcr->file_list) {
156 if (elt->seen || plugin_check_file(jcr, elt->fname)) {
159 Dmsg2(dbglvl, "deleted fname=%s seen=%i\n", elt->fname, elt->seen);
160 /* TODO: skip the decode and use directly the lstat field */
161 decode_stat(elt->lstat, &statc, &LinkFIc); /* decode catalog stat */
162 ff_pkt->fname = elt->fname;
163 ff_pkt->statp.st_mtime = statc.st_mtime;
164 ff_pkt->statp.st_ctime = statc.st_ctime;
165 encode_and_send_attributes(jcr, ff_pkt, stream);
169 term_find_files(ff_pkt);
173 void accurate_free(JCR *jcr)
175 if (jcr->file_list) {
176 jcr->file_list->destroy();
177 free(jcr->file_list);
178 jcr->file_list = NULL;
182 /* Send the deleted or the base file list and cleanup */
183 bool accurate_finish(JCR *jcr)
187 if (jcr->getJobLevel() == L_FULL) {
188 ret = accurate_send_base_file_list(jcr);
190 ret = accurate_send_deleted_list(jcr);
194 if (jcr->getJobLevel() == L_FULL) {
195 Jmsg(jcr, M_INFO, 0, _("Space saved with Base jobs: %lld MB\n"),
196 jcr->base_size/(1024*1024));
202 static bool accurate_add_file(JCR *jcr, uint32_t len,
203 char *fname, char *lstat, char *chksum,
211 /* we store CurFile, fname and ctime/mtime in the same chunk */
212 item = (CurFile *)jcr->file_list->hash_malloc(sizeof(CurFile)+len+3);
213 memcpy(item, &elt, sizeof(CurFile));
215 item->fname = (char *)item+sizeof(CurFile);
216 strcpy(item->fname, fname);
218 item->lstat = item->fname+strlen(item->fname)+1;
219 strcpy(item->lstat, lstat);
221 item->chksum = item->lstat+strlen(item->lstat)+1;
222 strcpy(item->chksum, chksum);
224 item->delta_seq = delta;
226 jcr->file_list->insert(item->fname, item);
228 Dmsg4(dbglvl, "add fname=<%s> lstat=%s delta_seq=%i chksum=%s\n",
229 fname, lstat, delta, chksum);
234 * This function is called for each file seen in fileset.
235 * We check in file_list hash if fname have been backuped
236 * the last time. After we can compare Lstat field.
237 * Full Lstat usage have been removed on 6612
239 * Returns: true if file has changed (must be backed up)
240 * false file not changed
242 bool accurate_check_file(JCR *jcr, FF_PKT *ff_pkt)
244 int digest_stream = STREAM_NONE;
245 DIGEST *digest = NULL;
254 ff_pkt->delta_seq = 0;
256 if (!jcr->accurate) {
262 if (S_ISDIR(ff_pkt->statp.st_mode)) {
263 fname = ff_pkt->link;
265 fname = ff_pkt->fname;
268 if (!accurate_lookup(jcr, fname, &elt)) {
269 Dmsg1(dbglvl, "accurate %s (not found)\n", fname);
274 ff_pkt->delta_seq = elt.delta_seq;
275 Dmsg1(10, "accurate delta_seq=%i\n", ff_pkt->delta_seq);
277 if (elt.seen) { /* file has been seen ? */
278 Dmsg1(dbglvl, "accurate %s (already seen)\n", fname);
282 decode_stat(elt.lstat, &statc, &LinkFIc); /* decode catalog stat */
284 if (jcr->getJobLevel() == L_FULL) {
285 opts = ff_pkt->BaseJobOpts;
287 opts = ff_pkt->AccurateOpts;
291 * Loop over options supplied by user and verify the
292 * fields he requests.
294 for (char *p=opts; !stat && *p; p++) {
295 char ed1[30], ed2[30];
297 case 'i': /* compare INODEs */
298 if (statc.st_ino != ff_pkt->statp.st_ino) {
299 Dmsg3(dbglvl-1, "%s st_ino differ. Cat: %s File: %s\n",
301 edit_uint64((uint64_t)statc.st_ino, ed1),
302 edit_uint64((uint64_t)ff_pkt->statp.st_ino, ed2));
306 case 'p': /* permissions bits */
307 /* TODO: If something change only in perm, user, group
308 * Backup only the attribute stream
310 if (statc.st_mode != ff_pkt->statp.st_mode) {
311 Dmsg3(dbglvl-1, "%s st_mode differ. Cat: %x File: %x\n",
313 (uint32_t)statc.st_mode, (uint32_t)ff_pkt->statp.st_mode);
317 case 'n': /* number of links */
318 if (statc.st_nlink != ff_pkt->statp.st_nlink) {
319 Dmsg3(dbglvl-1, "%s st_nlink differ. Cat: %d File: %d\n",
321 (uint32_t)statc.st_nlink, (uint32_t)ff_pkt->statp.st_nlink);
325 case 'u': /* user id */
326 if (statc.st_uid != ff_pkt->statp.st_uid) {
327 Dmsg3(dbglvl-1, "%s st_uid differ. Cat: %u File: %u\n",
329 (uint32_t)statc.st_uid, (uint32_t)ff_pkt->statp.st_uid);
333 case 'g': /* group id */
334 if (statc.st_gid != ff_pkt->statp.st_gid) {
335 Dmsg3(dbglvl-1, "%s st_gid differ. Cat: %u File: %u\n",
337 (uint32_t)statc.st_gid, (uint32_t)ff_pkt->statp.st_gid);
342 if (statc.st_size != ff_pkt->statp.st_size) {
343 Dmsg3(dbglvl-1, "%s st_size differ. Cat: %s File: %s\n",
345 edit_uint64((uint64_t)statc.st_size, ed1),
346 edit_uint64((uint64_t)ff_pkt->statp.st_size, ed2));
350 case 'a': /* access time */
351 if (statc.st_atime != ff_pkt->statp.st_atime) {
352 Dmsg1(dbglvl-1, "%s st_atime differs\n", fname);
356 case 'm': /* modification time */
357 if (statc.st_mtime != ff_pkt->statp.st_mtime) {
358 Dmsg1(dbglvl-1, "%s st_mtime differs\n", fname);
362 case 'c': /* ctime */
363 if (statc.st_ctime != ff_pkt->statp.st_ctime) {
364 Dmsg1(dbglvl-1, "%s st_ctime differs\n", fname);
368 case 'd': /* file size decrease */
369 if (statc.st_size > ff_pkt->statp.st_size) {
370 Dmsg3(dbglvl-1, "%s st_size decrease. Cat: %s File: %s\n",
372 edit_uint64((uint64_t)statc.st_size, ed1),
373 edit_uint64((uint64_t)ff_pkt->statp.st_size, ed2));
378 /* TODO: cleanup and factorise this function with verify.c */
379 case '5': /* compare MD5 */
380 case '1': /* compare SHA1 */
382 * The remainder of the function is all about getting the checksum.
383 * First we initialise, then we read files, other streams and Finder Info.
385 if (!stat && ff_pkt->type != FT_LNKSAVED &&
386 (S_ISREG(ff_pkt->statp.st_mode) &&
387 ff_pkt->flags & (FO_MD5|FO_SHA1|FO_SHA256|FO_SHA512)))
391 Jmsg(jcr, M_WARNING, 0, _("Can't verify checksum for %s\n"),
398 * Create our digest context. If this fails, the digest will be set
399 * to NULL and not used.
401 if (ff_pkt->flags & FO_MD5) {
402 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_MD5);
403 digest_stream = STREAM_MD5_DIGEST;
405 } else if (ff_pkt->flags & FO_SHA1) {
406 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA1);
407 digest_stream = STREAM_SHA1_DIGEST;
409 } else if (ff_pkt->flags & FO_SHA256) {
410 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA256);
411 digest_stream = STREAM_SHA256_DIGEST;
413 } else if (ff_pkt->flags & FO_SHA512) {
414 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA512);
415 digest_stream = STREAM_SHA512_DIGEST;
418 /* Did digest initialization fail? */
419 if (digest_stream != STREAM_NONE && digest == NULL) {
420 Jmsg(jcr, M_WARNING, 0, _("%s digest initialization failed\n"),
421 stream_to_ascii(digest_stream));
424 /* compute MD5 or SHA1 hash */
426 char md[CRYPTO_DIGEST_MAX_SIZE];
431 if (digest_file(jcr, ff_pkt, digest) != 0) {
434 } else if (crypto_digest_finalize(digest, (uint8_t *)md, &size)) {
436 const char *digest_name;
438 digest_buf = (char *)malloc(BASE64_SIZE(size));
439 digest_name = crypto_digest_name(digest);
441 bin_to_base64(digest_buf, BASE64_SIZE(size), md, size, true);
443 if (strcmp(digest_buf, elt.chksum)) {
444 Dmsg3(dbglvl-1, "%s chksum diff. Cat: %s File: %s\n",
453 crypto_digest_free(digest);
466 /* In Incr/Diff accurate mode, we mark all files as seen
467 * When in Full+Base mode, we mark only if the file match exactly
469 if (jcr->getJobLevel() == L_FULL) {
471 /* compute space saved with basefile */
472 jcr->base_size += ff_pkt->statp.st_size;
473 accurate_mark_file_as_seen(jcr, &elt);
476 accurate_mark_file_as_seen(jcr, &elt);
480 unstrip_path(ff_pkt);
485 * TODO: use big buffer from htable
487 int accurate_cmd(JCR *jcr)
489 BSOCK *dir = jcr->dir_bsock;
490 int lstat_pos, chksum_pos;
494 if (job_canceled(jcr)) {
497 if (sscanf(dir->msg, "accurate files=%ld", &nb) != 1) {
498 dir->fsend(_("2991 Bad accurate command\n"));
502 jcr->accurate = true;
504 accurate_init(jcr, nb);
507 * buffer = sizeof(CurFile) + dirmsg
508 * dirmsg = fname + \0 + lstat + \0 + checksum + \0 + delta_seq + \0
510 /* get current files */
511 while (dir->recv() >= 0) {
512 lstat_pos = strlen(dir->msg) + 1;
513 if (lstat_pos < dir->msglen) {
514 chksum_pos = lstat_pos + strlen(dir->msg + lstat_pos) + 1;
516 if (chksum_pos >= dir->msglen) {
517 chksum_pos = lstat_pos - 1; /* tweak: no checksum, point to the last \0 */
520 delta_seq = str_to_int32(dir->msg +
522 strlen(dir->msg + chksum_pos) + 1);
525 accurate_add_file(jcr, dir->msglen,
527 dir->msg + lstat_pos, /* LStat */
528 dir->msg + chksum_pos, /* CheckSum */
529 delta_seq); /* Delta Sequence */
534 extern void *start_heap;
536 char b1[50], b2[50], b3[50], b4[50], b5[50];
537 Dmsg5(dbglvl," Heap: heap=%s smbytes=%s max_bytes=%s bufs=%s max_bufs=%s\n",
538 edit_uint64_with_commas((char *)sbrk(0)-(char *)start_heap, b1),
539 edit_uint64_with_commas(sm_bytes, b2),
540 edit_uint64_with_commas(sm_max_bytes, b3),
541 edit_uint64_with_commas(sm_buffers, b4),
542 edit_uint64_with_commas(sm_max_buffers, b5));