2 Bacula® - The Network Backup Solution
4 Copyright (C) 2000-2011 Free Software Foundation Europe e.V.
6 The main author of Bacula is Kern Sibbald, with contributions from
7 many others, a complete list can be found in the file AUTHORS.
8 This program is Free Software; you can redistribute it and/or
9 modify it under the terms of version three of the GNU Affero General Public
10 License as published by the Free Software Foundation and included
13 This program is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 General Public License for more details.
18 You should have received a copy of the GNU Affero General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
23 Bacula® is a registered trademark of Kern Sibbald.
24 The licensor of Bacula is the Free Software Foundation Europe
25 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
26 Switzerland, email:ftf@fsfeurope.org.
36 static int dbglvl=100;
38 typedef struct PrivateCurFile {
47 bool accurate_mark_file_as_seen(JCR *jcr, char *fname)
49 if (!jcr->accurate || !jcr->file_list) {
52 /* TODO: just use elt->seen = 1 */
53 CurFile *temp = (CurFile *)jcr->file_list->lookup(fname);
55 temp->seen = 1; /* records are in memory */
56 Dmsg1(dbglvl, "marked <%s> as seen\n", fname);
58 Dmsg1(dbglvl, "<%s> not found to be marked as seen\n", fname);
63 static bool accurate_mark_file_as_seen(JCR *jcr, CurFile *elt)
65 /* TODO: just use elt->seen = 1 */
66 CurFile *temp = (CurFile *)jcr->file_list->lookup(elt->fname);
68 temp->seen = 1; /* records are in memory */
73 static bool accurate_lookup(JCR *jcr, char *fname, CurFile *ret)
78 CurFile *temp = (CurFile *)jcr->file_list->lookup(fname);
80 memcpy(ret, temp, sizeof(CurFile));
82 Dmsg1(dbglvl, "lookup <%s> ok\n", fname);
88 static bool accurate_init(JCR *jcr, int nbfile)
91 jcr->file_list = (htable *)malloc(sizeof(htable));
92 jcr->file_list->init(elt, &elt->link, nbfile);
96 static bool accurate_send_base_file_list(JCR *jcr)
102 int stream = STREAM_UNIX_ATTRIBUTES;
104 if (!jcr->accurate || jcr->getJobLevel() != L_FULL) {
108 if (jcr->file_list == NULL) {
112 ff_pkt = init_find_files();
113 ff_pkt->type = FT_BASE;
115 foreach_htable(elt, jcr->file_list) {
117 Dmsg2(dbglvl, "base file fname=%s seen=%i\n", elt->fname, elt->seen);
118 /* TODO: skip the decode and use directly the lstat field */
119 decode_stat(elt->lstat, &statc, &LinkFIc); /* decode catalog stat */
120 ff_pkt->fname = elt->fname;
121 ff_pkt->statp = statc;
122 encode_and_send_attributes(jcr, ff_pkt, stream);
127 term_find_files(ff_pkt);
132 /* This function is called at the end of backup
133 * We walk over all hash disk element, and we check
136 static bool accurate_send_deleted_list(JCR *jcr)
142 int stream = STREAM_UNIX_ATTRIBUTES;
144 if (!jcr->accurate) {
148 if (jcr->file_list == NULL) {
152 ff_pkt = init_find_files();
153 ff_pkt->type = FT_DELETED;
155 foreach_htable(elt, jcr->file_list) {
156 if (elt->seen || plugin_check_file(jcr, elt->fname)) {
159 Dmsg2(dbglvl, "deleted fname=%s seen=%i\n", elt->fname, elt->seen);
160 /* TODO: skip the decode and use directly the lstat field */
161 decode_stat(elt->lstat, &statc, &LinkFIc); /* decode catalog stat */
162 ff_pkt->fname = elt->fname;
163 ff_pkt->statp.st_mtime = statc.st_mtime;
164 ff_pkt->statp.st_ctime = statc.st_ctime;
165 encode_and_send_attributes(jcr, ff_pkt, stream);
169 term_find_files(ff_pkt);
173 void accurate_free(JCR *jcr)
175 if (jcr->file_list) {
176 jcr->file_list->destroy();
177 free(jcr->file_list);
178 jcr->file_list = NULL;
182 /* Send the deleted or the base file list and cleanup */
183 bool accurate_finish(JCR *jcr)
187 if (jcr->is_canceled() || jcr->is_incomplete()) {
192 if (jcr->is_JobLevel(L_FULL)) {
193 if (!jcr->rerunning) {
194 ret = accurate_send_base_file_list(jcr);
197 ret = accurate_send_deleted_list(jcr);
200 if (jcr->is_JobLevel(L_FULL)) {
201 Jmsg(jcr, M_INFO, 0, _("Space saved with Base jobs: %lld MB\n"),
202 jcr->base_size/(1024*1024));
208 static bool accurate_add_file(JCR *jcr, uint32_t len,
209 char *fname, char *lstat, char *chksum,
215 /* we store CurFile, fname and ctime/mtime in the same chunk */
216 item = (CurFile *)jcr->file_list->hash_malloc(sizeof(CurFile)+len);
219 /* TODO: see if we can optimize this part with memcpy instead of strcpy */
220 item->fname = (char *)item+sizeof(CurFile);
221 strcpy(item->fname, fname);
223 item->lstat = item->fname+strlen(item->fname)+1;
224 strcpy(item->lstat, lstat);
226 item->chksum = item->lstat+strlen(item->lstat)+1;
227 strcpy(item->chksum, chksum);
229 item->delta_seq = delta;
231 jcr->file_list->insert(item->fname, item);
233 Dmsg4(dbglvl, "add fname=<%s> lstat=%s delta_seq=%i chksum=%s\n",
234 fname, lstat, delta, chksum);
239 * This function is called for each file seen in fileset.
240 * We check in file_list hash if fname have been backuped
241 * the last time. After we can compare Lstat field.
242 * Full Lstat usage have been removed on 6612
244 * Returns: true if file has changed (must be backed up)
245 * false file not changed
247 bool accurate_check_file(JCR *jcr, FF_PKT *ff_pkt)
249 int digest_stream = STREAM_NONE;
250 DIGEST *digest = NULL;
259 ff_pkt->delta_seq = 0;
261 if (!jcr->accurate && !jcr->rerunning) {
267 if (S_ISDIR(ff_pkt->statp.st_mode)) {
268 fname = ff_pkt->link;
270 fname = ff_pkt->fname;
273 if (!accurate_lookup(jcr, fname, &elt)) {
274 Dmsg1(dbglvl, "accurate %s (not found)\n", fname);
279 ff_pkt->delta_seq = elt.delta_seq;
281 if (elt.seen) { /* file has been seen ? */
282 Dmsg1(dbglvl, "accurate %s (already seen)\n", fname);
286 decode_stat(elt.lstat, &statc, &LinkFIc); /* decode catalog stat */
288 if (!jcr->rerunning && (jcr->getJobLevel() == L_FULL)) {
289 opts = ff_pkt->BaseJobOpts;
291 opts = ff_pkt->AccurateOpts;
295 * Loop over options supplied by user and verify the
296 * fields he requests.
298 for (char *p=opts; !stat && *p; p++) {
299 char ed1[30], ed2[30];
301 case 'i': /* compare INODEs */
302 if (statc.st_ino != ff_pkt->statp.st_ino) {
303 Dmsg3(dbglvl-1, "%s st_ino differ. Cat: %s File: %s\n",
305 edit_uint64((uint64_t)statc.st_ino, ed1),
306 edit_uint64((uint64_t)ff_pkt->statp.st_ino, ed2));
310 case 'p': /* permissions bits */
311 /* TODO: If something change only in perm, user, group
312 * Backup only the attribute stream
314 if (statc.st_mode != ff_pkt->statp.st_mode) {
315 Dmsg3(dbglvl-1, "%s st_mode differ. Cat: %x File: %x\n",
317 (uint32_t)statc.st_mode, (uint32_t)ff_pkt->statp.st_mode);
321 case 'n': /* number of links */
322 if (statc.st_nlink != ff_pkt->statp.st_nlink) {
323 Dmsg3(dbglvl-1, "%s st_nlink differ. Cat: %d File: %d\n",
325 (uint32_t)statc.st_nlink, (uint32_t)ff_pkt->statp.st_nlink);
329 case 'u': /* user id */
330 if (statc.st_uid != ff_pkt->statp.st_uid) {
331 Dmsg3(dbglvl-1, "%s st_uid differ. Cat: %u File: %u\n",
333 (uint32_t)statc.st_uid, (uint32_t)ff_pkt->statp.st_uid);
337 case 'g': /* group id */
338 if (statc.st_gid != ff_pkt->statp.st_gid) {
339 Dmsg3(dbglvl-1, "%s st_gid differ. Cat: %u File: %u\n",
341 (uint32_t)statc.st_gid, (uint32_t)ff_pkt->statp.st_gid);
346 if (statc.st_size != ff_pkt->statp.st_size) {
347 Dmsg3(dbglvl-1, "%s st_size differ. Cat: %s File: %s\n",
349 edit_uint64((uint64_t)statc.st_size, ed1),
350 edit_uint64((uint64_t)ff_pkt->statp.st_size, ed2));
354 case 'a': /* access time */
355 if (statc.st_atime != ff_pkt->statp.st_atime) {
356 Dmsg1(dbglvl-1, "%s st_atime differs\n", fname);
360 case 'm': /* modification time */
361 if (statc.st_mtime != ff_pkt->statp.st_mtime) {
362 Dmsg1(dbglvl-1, "%s st_mtime differs\n", fname);
366 case 'c': /* ctime */
367 if (statc.st_ctime != ff_pkt->statp.st_ctime) {
368 Dmsg1(dbglvl-1, "%s st_ctime differs\n", fname);
372 case 'd': /* file size decrease */
373 if (statc.st_size > ff_pkt->statp.st_size) {
374 Dmsg3(dbglvl-1, "%s st_size decrease. Cat: %s File: %s\n",
376 edit_uint64((uint64_t)statc.st_size, ed1),
377 edit_uint64((uint64_t)ff_pkt->statp.st_size, ed2));
382 /* TODO: cleanup and factorise this function with verify.c */
383 case '5': /* compare MD5 */
384 case '1': /* compare SHA1 */
386 * The remainder of the function is all about getting the checksum.
387 * First we initialise, then we read files, other streams and Finder Info.
389 if (!stat && ff_pkt->type != FT_LNKSAVED &&
390 (S_ISREG(ff_pkt->statp.st_mode) &&
391 ff_pkt->flags & (FO_MD5|FO_SHA1|FO_SHA256|FO_SHA512)))
394 if (!*elt.chksum && !jcr->rerunning) {
395 Jmsg(jcr, M_WARNING, 0, _("Cannot verify checksum for %s\n"),
402 * Create our digest context. If this fails, the digest will be set
403 * to NULL and not used.
405 if (ff_pkt->flags & FO_MD5) {
406 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_MD5);
407 digest_stream = STREAM_MD5_DIGEST;
409 } else if (ff_pkt->flags & FO_SHA1) {
410 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA1);
411 digest_stream = STREAM_SHA1_DIGEST;
413 } else if (ff_pkt->flags & FO_SHA256) {
414 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA256);
415 digest_stream = STREAM_SHA256_DIGEST;
417 } else if (ff_pkt->flags & FO_SHA512) {
418 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA512);
419 digest_stream = STREAM_SHA512_DIGEST;
422 /* Did digest initialization fail? */
423 if (digest_stream != STREAM_NONE && digest == NULL) {
424 Jmsg(jcr, M_WARNING, 0, _("%s digest initialization failed\n"),
425 stream_to_ascii(digest_stream));
428 /* compute MD5 or SHA1 hash */
430 char md[CRYPTO_DIGEST_MAX_SIZE];
435 if (digest_file(jcr, ff_pkt, digest) != 0) {
438 } else if (crypto_digest_finalize(digest, (uint8_t *)md, &size)) {
440 const char *digest_name;
442 digest_buf = (char *)malloc(BASE64_SIZE(size));
443 digest_name = crypto_digest_name(digest);
445 bin_to_base64(digest_buf, BASE64_SIZE(size), md, size, true);
447 if (strcmp(digest_buf, elt.chksum)) {
448 Dmsg4(dbglvl,"%s %s chksum diff. Cat: %s File: %s\n",
458 crypto_digest_free(digest);
471 /* In Incr/Diff accurate mode, we mark all files as seen
472 * When in Full+Base mode, we mark only if the file match exactly
474 if (jcr->getJobLevel() == L_FULL) {
476 /* compute space saved with basefile */
477 jcr->base_size += ff_pkt->statp.st_size;
478 accurate_mark_file_as_seen(jcr, &elt);
481 accurate_mark_file_as_seen(jcr, &elt);
485 unstrip_path(ff_pkt);
490 * TODO: use big buffer from htable
492 int accurate_cmd(JCR *jcr)
494 BSOCK *dir = jcr->dir_bsock;
495 int lstat_pos, chksum_pos;
499 if (job_canceled(jcr)) {
502 if (sscanf(dir->msg, "accurate files=%ld", &nb) != 1) {
503 dir->fsend(_("2991 Bad accurate command\n"));
507 jcr->accurate = true;
509 accurate_init(jcr, nb);
512 * buffer = sizeof(CurFile) + dirmsg
513 * dirmsg = fname + \0 + lstat + \0 + checksum + \0 + delta_seq + \0
515 /* get current files */
516 while (dir->recv() >= 0) {
517 lstat_pos = strlen(dir->msg) + 1;
518 if (lstat_pos < dir->msglen) {
519 chksum_pos = lstat_pos + strlen(dir->msg + lstat_pos) + 1;
521 if (chksum_pos >= dir->msglen) {
522 chksum_pos = lstat_pos - 1; /* tweak: no checksum, point to the last \0 */
525 delta_seq = str_to_int32(dir->msg +
527 strlen(dir->msg + chksum_pos) + 1);
530 accurate_add_file(jcr, dir->msglen,
532 dir->msg + lstat_pos, /* LStat */
533 dir->msg + chksum_pos, /* CheckSum */
534 delta_seq); /* Delta Sequence */
539 extern void *start_heap;
541 char b1[50], b2[50], b3[50], b4[50], b5[50];
542 Dmsg5(dbglvl," Heap: heap=%s smbytes=%s max_bytes=%s bufs=%s max_bufs=%s\n",
543 edit_uint64_with_commas((char *)sbrk(0)-(char *)start_heap, b1),
544 edit_uint64_with_commas(sm_bytes, b2),
545 edit_uint64_with_commas(sm_max_bytes, b3),
546 edit_uint64_with_commas(sm_buffers, b4),
547 edit_uint64_with_commas(sm_max_buffers, b5));