2 Bacula® - The Network Backup Solution
4 Copyright (C) 2000-2009 Free Software Foundation Europe e.V.
6 The main author of Bacula is Kern Sibbald, with contributions from
7 many others, a complete list can be found in the file AUTHORS.
8 This program is Free Software; you can redistribute it and/or
9 modify it under the terms of version three of the GNU Affero General Public
10 License as published by the Free Software Foundation and included
13 This program is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 General Public License for more details.
18 You should have received a copy of the GNU Affero General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
23 Bacula® is a registered trademark of Kern Sibbald.
24 The licensor of Bacula is the Free Software Foundation Europe
25 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
26 Switzerland, email:ftf@fsfeurope.org.
36 static int dbglvl=100;
38 typedef struct PrivateCurFile {
47 bool accurate_mark_file_as_seen(JCR *jcr, char *fname)
49 if (!jcr->accurate || !jcr->file_list) {
52 /* TODO: just use elt->seen = 1 */
53 CurFile *temp = (CurFile *)jcr->file_list->lookup(fname);
55 temp->seen = 1; /* records are in memory */
56 Dmsg1(dbglvl, "marked <%s> as seen\n", fname);
58 Dmsg1(dbglvl, "<%s> not found to be marked as seen\n", fname);
63 static bool accurate_mark_file_as_seen(JCR *jcr, CurFile *elt)
65 /* TODO: just use elt->seen = 1 */
66 CurFile *temp = (CurFile *)jcr->file_list->lookup(elt->fname);
68 temp->seen = 1; /* records are in memory */
73 static bool accurate_lookup(JCR *jcr, char *fname, CurFile *ret)
78 CurFile *temp = (CurFile *)jcr->file_list->lookup(fname);
80 memcpy(ret, temp, sizeof(CurFile));
82 Dmsg1(dbglvl, "lookup <%s> ok\n", fname);
88 static bool accurate_init(JCR *jcr, int nbfile)
91 jcr->file_list = (htable *)malloc(sizeof(htable));
92 jcr->file_list->init(elt, &elt->link, nbfile);
96 static bool accurate_send_base_file_list(JCR *jcr)
102 int stream = STREAM_UNIX_ATTRIBUTES;
104 if (!jcr->accurate || jcr->getJobLevel() != L_FULL) {
108 if (jcr->file_list == NULL) {
112 ff_pkt = init_find_files();
113 ff_pkt->type = FT_BASE;
115 foreach_htable(elt, jcr->file_list) {
117 Dmsg2(dbglvl, "base file fname=%s seen=%i\n", elt->fname, elt->seen);
118 /* TODO: skip the decode and use directly the lstat field */
119 decode_stat(elt->lstat, &statc, &LinkFIc); /* decode catalog stat */
120 ff_pkt->fname = elt->fname;
121 ff_pkt->statp = statc;
122 encode_and_send_attributes(jcr, ff_pkt, stream);
127 term_find_files(ff_pkt);
132 /* This function is called at the end of backup
133 * We walk over all hash disk element, and we check
136 static bool accurate_send_deleted_list(JCR *jcr)
142 int stream = STREAM_UNIX_ATTRIBUTES;
144 if (!jcr->accurate) {
148 if (jcr->file_list == NULL) {
152 ff_pkt = init_find_files();
153 ff_pkt->type = FT_DELETED;
155 foreach_htable(elt, jcr->file_list) {
156 if (elt->seen || plugin_check_file(jcr, elt->fname)) {
159 Dmsg2(dbglvl, "deleted fname=%s seen=%i\n", elt->fname, elt->seen);
160 /* TODO: skip the decode and use directly the lstat field */
161 decode_stat(elt->lstat, &statc, &LinkFIc); /* decode catalog stat */
162 ff_pkt->fname = elt->fname;
163 ff_pkt->statp.st_mtime = statc.st_mtime;
164 ff_pkt->statp.st_ctime = statc.st_ctime;
165 encode_and_send_attributes(jcr, ff_pkt, stream);
169 term_find_files(ff_pkt);
173 void accurate_free(JCR *jcr)
175 if (jcr->file_list) {
176 jcr->file_list->destroy();
177 free(jcr->file_list);
178 jcr->file_list = NULL;
182 /* Send the deleted or the base file list and cleanup */
183 bool accurate_finish(JCR *jcr)
187 if (jcr->getJobLevel() == L_FULL) {
188 ret = accurate_send_base_file_list(jcr);
190 ret = accurate_send_deleted_list(jcr);
194 if (jcr->getJobLevel() == L_FULL) {
195 Jmsg(jcr, M_INFO, 0, _("Space saved with Base jobs: %lld MB\n"),
196 jcr->base_size/(1024*1024));
202 static bool accurate_add_file(JCR *jcr, uint32_t len,
203 char *fname, char *lstat, char *chksum,
209 /* we store CurFile, fname and ctime/mtime in the same chunk */
210 /* TODO: see if len contains already the 3 \0 */
211 item = (CurFile *)jcr->file_list->hash_malloc(sizeof(CurFile)+len+3);
214 /* TODO: see if we can optimize this part with memcpy instead of strcpy */
215 item->fname = (char *)item+sizeof(CurFile);
216 strcpy(item->fname, fname);
218 item->lstat = item->fname+strlen(item->fname)+1;
219 strcpy(item->lstat, lstat);
221 item->chksum = item->lstat+strlen(item->lstat)+1;
222 strcpy(item->chksum, chksum);
224 item->delta_seq = delta;
226 jcr->file_list->insert(item->fname, item);
228 Dmsg4(dbglvl, "add fname=<%s> lstat=%s delta_seq=%i chksum=%s\n",
229 fname, lstat, delta, chksum);
234 * This function is called for each file seen in fileset.
235 * We check in file_list hash if fname have been backuped
236 * the last time. After we can compare Lstat field.
237 * Full Lstat usage have been removed on 6612
239 * Returns: true if file has changed (must be backed up)
240 * false file not changed
242 bool accurate_check_file(JCR *jcr, FF_PKT *ff_pkt)
244 int digest_stream = STREAM_NONE;
245 DIGEST *digest = NULL;
254 ff_pkt->delta_seq = 0;
256 if (!jcr->accurate) {
262 if (S_ISDIR(ff_pkt->statp.st_mode)) {
263 fname = ff_pkt->link;
265 fname = ff_pkt->fname;
268 if (!accurate_lookup(jcr, fname, &elt)) {
269 Dmsg1(dbglvl, "accurate %s (not found)\n", fname);
274 ff_pkt->delta_seq = elt.delta_seq;
276 if (elt.seen) { /* file has been seen ? */
277 Dmsg1(dbglvl, "accurate %s (already seen)\n", fname);
281 decode_stat(elt.lstat, &statc, &LinkFIc); /* decode catalog stat */
283 if (jcr->getJobLevel() == L_FULL) {
284 opts = ff_pkt->BaseJobOpts;
286 opts = ff_pkt->AccurateOpts;
290 * Loop over options supplied by user and verify the
291 * fields he requests.
293 for (char *p=opts; !stat && *p; p++) {
294 char ed1[30], ed2[30];
296 case 'i': /* compare INODEs */
297 if (statc.st_ino != ff_pkt->statp.st_ino) {
298 Dmsg3(dbglvl-1, "%s st_ino differ. Cat: %s File: %s\n",
300 edit_uint64((uint64_t)statc.st_ino, ed1),
301 edit_uint64((uint64_t)ff_pkt->statp.st_ino, ed2));
305 case 'p': /* permissions bits */
306 /* TODO: If something change only in perm, user, group
307 * Backup only the attribute stream
309 if (statc.st_mode != ff_pkt->statp.st_mode) {
310 Dmsg3(dbglvl-1, "%s st_mode differ. Cat: %x File: %x\n",
312 (uint32_t)statc.st_mode, (uint32_t)ff_pkt->statp.st_mode);
316 case 'n': /* number of links */
317 if (statc.st_nlink != ff_pkt->statp.st_nlink) {
318 Dmsg3(dbglvl-1, "%s st_nlink differ. Cat: %d File: %d\n",
320 (uint32_t)statc.st_nlink, (uint32_t)ff_pkt->statp.st_nlink);
324 case 'u': /* user id */
325 if (statc.st_uid != ff_pkt->statp.st_uid) {
326 Dmsg3(dbglvl-1, "%s st_uid differ. Cat: %u File: %u\n",
328 (uint32_t)statc.st_uid, (uint32_t)ff_pkt->statp.st_uid);
332 case 'g': /* group id */
333 if (statc.st_gid != ff_pkt->statp.st_gid) {
334 Dmsg3(dbglvl-1, "%s st_gid differ. Cat: %u File: %u\n",
336 (uint32_t)statc.st_gid, (uint32_t)ff_pkt->statp.st_gid);
341 if (statc.st_size != ff_pkt->statp.st_size) {
342 Dmsg3(dbglvl-1, "%s st_size differ. Cat: %s File: %s\n",
344 edit_uint64((uint64_t)statc.st_size, ed1),
345 edit_uint64((uint64_t)ff_pkt->statp.st_size, ed2));
349 case 'a': /* access time */
350 if (statc.st_atime != ff_pkt->statp.st_atime) {
351 Dmsg1(dbglvl-1, "%s st_atime differs\n", fname);
355 case 'm': /* modification time */
356 if (statc.st_mtime != ff_pkt->statp.st_mtime) {
357 Dmsg1(dbglvl-1, "%s st_mtime differs\n", fname);
361 case 'c': /* ctime */
362 if (statc.st_ctime != ff_pkt->statp.st_ctime) {
363 Dmsg1(dbglvl-1, "%s st_ctime differs\n", fname);
367 case 'd': /* file size decrease */
368 if (statc.st_size > ff_pkt->statp.st_size) {
369 Dmsg3(dbglvl-1, "%s st_size decrease. Cat: %s File: %s\n",
371 edit_uint64((uint64_t)statc.st_size, ed1),
372 edit_uint64((uint64_t)ff_pkt->statp.st_size, ed2));
377 /* TODO: cleanup and factorise this function with verify.c */
378 case '5': /* compare MD5 */
379 case '1': /* compare SHA1 */
381 * The remainder of the function is all about getting the checksum.
382 * First we initialise, then we read files, other streams and Finder Info.
384 if (!stat && ff_pkt->type != FT_LNKSAVED &&
385 (S_ISREG(ff_pkt->statp.st_mode) &&
386 ff_pkt->flags & (FO_MD5|FO_SHA1|FO_SHA256|FO_SHA512)))
390 Jmsg(jcr, M_WARNING, 0, _("Can't verify checksum for %s\n"),
397 * Create our digest context. If this fails, the digest will be set
398 * to NULL and not used.
400 if (ff_pkt->flags & FO_MD5) {
401 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_MD5);
402 digest_stream = STREAM_MD5_DIGEST;
404 } else if (ff_pkt->flags & FO_SHA1) {
405 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA1);
406 digest_stream = STREAM_SHA1_DIGEST;
408 } else if (ff_pkt->flags & FO_SHA256) {
409 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA256);
410 digest_stream = STREAM_SHA256_DIGEST;
412 } else if (ff_pkt->flags & FO_SHA512) {
413 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA512);
414 digest_stream = STREAM_SHA512_DIGEST;
417 /* Did digest initialization fail? */
418 if (digest_stream != STREAM_NONE && digest == NULL) {
419 Jmsg(jcr, M_WARNING, 0, _("%s digest initialization failed\n"),
420 stream_to_ascii(digest_stream));
423 /* compute MD5 or SHA1 hash */
425 char md[CRYPTO_DIGEST_MAX_SIZE];
430 if (digest_file(jcr, ff_pkt, digest) != 0) {
433 } else if (crypto_digest_finalize(digest, (uint8_t *)md, &size)) {
435 const char *digest_name;
437 digest_buf = (char *)malloc(BASE64_SIZE(size));
438 digest_name = crypto_digest_name(digest);
440 bin_to_base64(digest_buf, BASE64_SIZE(size), md, size, true);
442 if (strcmp(digest_buf, elt.chksum)) {
443 Dmsg3(dbglvl-1, "%s chksum diff. Cat: %s File: %s\n",
452 crypto_digest_free(digest);
465 /* In Incr/Diff accurate mode, we mark all files as seen
466 * When in Full+Base mode, we mark only if the file match exactly
468 if (jcr->getJobLevel() == L_FULL) {
470 /* compute space saved with basefile */
471 jcr->base_size += ff_pkt->statp.st_size;
472 accurate_mark_file_as_seen(jcr, &elt);
475 accurate_mark_file_as_seen(jcr, &elt);
479 unstrip_path(ff_pkt);
484 * TODO: use big buffer from htable
486 int accurate_cmd(JCR *jcr)
488 BSOCK *dir = jcr->dir_bsock;
489 int lstat_pos, chksum_pos;
493 if (job_canceled(jcr)) {
496 if (sscanf(dir->msg, "accurate files=%ld", &nb) != 1) {
497 dir->fsend(_("2991 Bad accurate command\n"));
501 jcr->accurate = true;
503 accurate_init(jcr, nb);
506 * buffer = sizeof(CurFile) + dirmsg
507 * dirmsg = fname + \0 + lstat + \0 + checksum + \0 + delta_seq + \0
509 /* get current files */
510 while (dir->recv() >= 0) {
511 lstat_pos = strlen(dir->msg) + 1;
512 if (lstat_pos < dir->msglen) {
513 chksum_pos = lstat_pos + strlen(dir->msg + lstat_pos) + 1;
515 if (chksum_pos >= dir->msglen) {
516 chksum_pos = lstat_pos - 1; /* tweak: no checksum, point to the last \0 */
519 delta_seq = str_to_int32(dir->msg +
521 strlen(dir->msg + chksum_pos) + 1);
524 accurate_add_file(jcr, dir->msglen,
526 dir->msg + lstat_pos, /* LStat */
527 dir->msg + chksum_pos, /* CheckSum */
528 delta_seq); /* Delta Sequence */
533 extern void *start_heap;
535 char b1[50], b2[50], b3[50], b4[50], b5[50];
536 Dmsg5(dbglvl," Heap: heap=%s smbytes=%s max_bytes=%s bufs=%s max_bufs=%s\n",
537 edit_uint64_with_commas((char *)sbrk(0)-(char *)start_heap, b1),
538 edit_uint64_with_commas(sm_bytes, b2),
539 edit_uint64_with_commas(sm_max_bytes, b3),
540 edit_uint64_with_commas(sm_buffers, b4),
541 edit_uint64_with_commas(sm_max_buffers, b5));