2 Bacula® - The Network Backup Solution
4 Copyright (C) 2000-2011 Free Software Foundation Europe e.V.
6 The main author of Bacula is Kern Sibbald, with contributions from
7 many others, a complete list can be found in the file AUTHORS.
8 This program is Free Software; you can redistribute it and/or
9 modify it under the terms of version three of the GNU Affero General Public
10 License as published by the Free Software Foundation and included
13 This program is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 General Public License for more details.
18 You should have received a copy of the GNU Affero General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
23 Bacula® is a registered trademark of Kern Sibbald.
24 The licensor of Bacula is the Free Software Foundation Europe
25 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
26 Switzerland, email:ftf@fsfeurope.org.
36 static int dbglvl=100;
38 typedef struct PrivateCurFile {
47 bool accurate_mark_file_as_seen(JCR *jcr, char *fname)
49 if (!jcr->accurate || !jcr->file_list) {
52 /* TODO: just use elt->seen = 1 */
53 CurFile *temp = (CurFile *)jcr->file_list->lookup(fname);
55 temp->seen = 1; /* records are in memory */
56 Dmsg1(dbglvl, "marked <%s> as seen\n", fname);
58 Dmsg1(dbglvl, "<%s> not found to be marked as seen\n", fname);
63 static bool accurate_mark_file_as_seen(JCR *jcr, CurFile *elt)
65 /* TODO: just use elt->seen = 1 */
66 CurFile *temp = (CurFile *)jcr->file_list->lookup(elt->fname);
68 temp->seen = 1; /* records are in memory */
73 static bool accurate_lookup(JCR *jcr, char *fname, CurFile *ret)
78 CurFile *temp = (CurFile *)jcr->file_list->lookup(fname);
80 memcpy(ret, temp, sizeof(CurFile));
82 Dmsg1(dbglvl, "lookup <%s> ok\n", fname);
88 static bool accurate_init(JCR *jcr, int nbfile)
91 jcr->file_list = (htable *)malloc(sizeof(htable));
92 jcr->file_list->init(elt, &elt->link, nbfile);
96 static bool accurate_send_base_file_list(JCR *jcr)
102 int stream = STREAM_UNIX_ATTRIBUTES;
104 if (!jcr->accurate || jcr->getJobLevel() != L_FULL) {
108 if (jcr->file_list == NULL) {
112 ff_pkt = init_find_files();
113 ff_pkt->type = FT_BASE;
115 foreach_htable(elt, jcr->file_list) {
117 Dmsg2(dbglvl, "base file fname=%s seen=%i\n", elt->fname, elt->seen);
118 /* TODO: skip the decode and use directly the lstat field */
119 decode_stat(elt->lstat, &statc, sizeof(statc), &LinkFIc); /* decode catalog stat */
120 ff_pkt->fname = elt->fname;
121 ff_pkt->statp = statc;
122 encode_and_send_attributes(jcr, ff_pkt, stream);
127 term_find_files(ff_pkt);
132 /* This function is called at the end of backup
133 * We walk over all hash disk element, and we check
136 static bool accurate_send_deleted_list(JCR *jcr)
142 int stream = STREAM_UNIX_ATTRIBUTES;
144 if (!jcr->accurate) {
148 if (jcr->file_list == NULL) {
152 ff_pkt = init_find_files();
153 ff_pkt->type = FT_DELETED;
155 foreach_htable(elt, jcr->file_list) {
156 if (elt->seen || plugin_check_file(jcr, elt->fname)) {
159 Dmsg2(dbglvl, "deleted fname=%s seen=%i\n", elt->fname, elt->seen);
160 /* TODO: skip the decode and use directly the lstat field */
161 decode_stat(elt->lstat, &statc, sizeof(statc), &LinkFIc); /* decode catalog stat */
162 ff_pkt->fname = elt->fname;
163 ff_pkt->statp.st_mtime = statc.st_mtime;
164 ff_pkt->statp.st_ctime = statc.st_ctime;
165 encode_and_send_attributes(jcr, ff_pkt, stream);
169 term_find_files(ff_pkt);
173 void accurate_free(JCR *jcr)
175 if (jcr->file_list) {
176 jcr->file_list->destroy();
177 free(jcr->file_list);
178 jcr->file_list = NULL;
182 /* Send the deleted or the base file list and cleanup */
183 bool accurate_finish(JCR *jcr)
187 if (jcr->is_canceled() || jcr->is_incomplete()) {
192 if (jcr->is_JobLevel(L_FULL)) {
193 if (!jcr->rerunning) {
194 ret = accurate_send_base_file_list(jcr);
197 ret = accurate_send_deleted_list(jcr);
200 if (jcr->is_JobLevel(L_FULL)) {
201 Jmsg(jcr, M_INFO, 0, _("Space saved with Base jobs: %lld MB\n"),
202 jcr->base_size/(1024*1024));
208 static bool accurate_add_file(JCR *jcr, uint32_t len,
209 char *fname, char *lstat, char *chksum,
215 /* we store CurFile, fname and ctime/mtime in the same chunk
216 * we need one extra byte to handle an empty chksum
218 item = (CurFile *)jcr->file_list->hash_malloc(sizeof(CurFile)+len+1);
221 /* TODO: see if we can optimize this part with memcpy instead of strcpy */
222 item->fname = (char *)item+sizeof(CurFile);
223 strcpy(item->fname, fname);
225 item->lstat = item->fname+strlen(item->fname)+1;
226 strcpy(item->lstat, lstat);
228 item->chksum = item->lstat+strlen(item->lstat)+1;
229 strcpy(item->chksum, chksum);
231 item->delta_seq = delta;
233 jcr->file_list->insert(item->fname, item);
235 Dmsg4(dbglvl, "add fname=<%s> lstat=%s delta_seq=%i chksum=%s\n",
236 fname, lstat, delta, chksum);
241 * This function is called for each file seen in fileset.
242 * We check in file_list hash if fname have been backuped
243 * the last time. After we can compare Lstat field.
244 * Full Lstat usage have been removed on 6612
246 * Returns: true if file has changed (must be backed up)
247 * false file not changed
249 bool accurate_check_file(JCR *jcr, FF_PKT *ff_pkt)
251 int digest_stream = STREAM_NONE;
252 DIGEST *digest = NULL;
261 ff_pkt->delta_seq = 0;
262 ff_pkt->accurate_found = false;
264 if (!jcr->accurate && !jcr->rerunning) {
270 if (S_ISDIR(ff_pkt->statp.st_mode)) {
271 fname = ff_pkt->link;
273 fname = ff_pkt->fname;
276 if (!accurate_lookup(jcr, fname, &elt)) {
277 Dmsg1(dbglvl, "accurate %s (not found)\n", fname);
282 ff_pkt->accurate_found = true;
283 ff_pkt->delta_seq = elt.delta_seq;
285 if (elt.seen) { /* file has been seen ? */
286 Dmsg1(dbglvl, "accurate %s (already seen)\n", fname);
290 decode_stat(elt.lstat, &statc, sizeof(statc), &LinkFIc); /* decode catalog stat */
292 if (!jcr->rerunning && (jcr->getJobLevel() == L_FULL)) {
293 opts = ff_pkt->BaseJobOpts;
295 opts = ff_pkt->AccurateOpts;
299 * Loop over options supplied by user and verify the
300 * fields he requests.
302 for (char *p=opts; !stat && *p; p++) {
303 char ed1[30], ed2[30];
305 case 'i': /* compare INODEs */
306 if (statc.st_ino != ff_pkt->statp.st_ino) {
307 Dmsg3(dbglvl-1, "%s st_ino differ. Cat: %s File: %s\n",
309 edit_uint64((uint64_t)statc.st_ino, ed1),
310 edit_uint64((uint64_t)ff_pkt->statp.st_ino, ed2));
314 case 'p': /* permissions bits */
315 /* TODO: If something change only in perm, user, group
316 * Backup only the attribute stream
318 if (statc.st_mode != ff_pkt->statp.st_mode) {
319 Dmsg3(dbglvl-1, "%s st_mode differ. Cat: %x File: %x\n",
321 (uint32_t)statc.st_mode, (uint32_t)ff_pkt->statp.st_mode);
325 case 'n': /* number of links */
326 if (statc.st_nlink != ff_pkt->statp.st_nlink) {
327 Dmsg3(dbglvl-1, "%s st_nlink differ. Cat: %d File: %d\n",
329 (uint32_t)statc.st_nlink, (uint32_t)ff_pkt->statp.st_nlink);
333 case 'u': /* user id */
334 if (statc.st_uid != ff_pkt->statp.st_uid) {
335 Dmsg3(dbglvl-1, "%s st_uid differ. Cat: %u File: %u\n",
337 (uint32_t)statc.st_uid, (uint32_t)ff_pkt->statp.st_uid);
341 case 'g': /* group id */
342 if (statc.st_gid != ff_pkt->statp.st_gid) {
343 Dmsg3(dbglvl-1, "%s st_gid differ. Cat: %u File: %u\n",
345 (uint32_t)statc.st_gid, (uint32_t)ff_pkt->statp.st_gid);
350 if (statc.st_size != ff_pkt->statp.st_size) {
351 Dmsg3(dbglvl-1, "%s st_size differ. Cat: %s File: %s\n",
353 edit_uint64((uint64_t)statc.st_size, ed1),
354 edit_uint64((uint64_t)ff_pkt->statp.st_size, ed2));
358 case 'a': /* access time */
359 if (statc.st_atime != ff_pkt->statp.st_atime) {
360 Dmsg1(dbglvl-1, "%s st_atime differs\n", fname);
364 case 'm': /* modification time */
365 if (statc.st_mtime != ff_pkt->statp.st_mtime) {
366 Dmsg1(dbglvl-1, "%s st_mtime differs\n", fname);
370 case 'c': /* ctime */
371 if (statc.st_ctime != ff_pkt->statp.st_ctime) {
372 Dmsg1(dbglvl-1, "%s st_ctime differs\n", fname);
376 case 'd': /* file size decrease */
377 if (statc.st_size > ff_pkt->statp.st_size) {
378 Dmsg3(dbglvl-1, "%s st_size decrease. Cat: %s File: %s\n",
380 edit_uint64((uint64_t)statc.st_size, ed1),
381 edit_uint64((uint64_t)ff_pkt->statp.st_size, ed2));
385 case 'A': /* Always backup a file */
388 /* TODO: cleanup and factorise this function with verify.c */
389 case '5': /* compare MD5 */
390 case '1': /* compare SHA1 */
392 * The remainder of the function is all about getting the checksum.
393 * First we initialise, then we read files, other streams and Finder Info.
395 if (!stat && ff_pkt->type != FT_LNKSAVED &&
396 (S_ISREG(ff_pkt->statp.st_mode) &&
397 ff_pkt->flags & (FO_MD5|FO_SHA1|FO_SHA256|FO_SHA512)))
400 if (!*elt.chksum && !jcr->rerunning) {
401 Jmsg(jcr, M_WARNING, 0, _("Cannot verify checksum for %s\n"),
408 * Create our digest context. If this fails, the digest will be set
409 * to NULL and not used.
411 if (ff_pkt->flags & FO_MD5) {
412 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_MD5);
413 digest_stream = STREAM_MD5_DIGEST;
415 } else if (ff_pkt->flags & FO_SHA1) {
416 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA1);
417 digest_stream = STREAM_SHA1_DIGEST;
419 } else if (ff_pkt->flags & FO_SHA256) {
420 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA256);
421 digest_stream = STREAM_SHA256_DIGEST;
423 } else if (ff_pkt->flags & FO_SHA512) {
424 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA512);
425 digest_stream = STREAM_SHA512_DIGEST;
428 /* Did digest initialization fail? */
429 if (digest_stream != STREAM_NONE && digest == NULL) {
430 Jmsg(jcr, M_WARNING, 0, _("%s digest initialization failed\n"),
431 stream_to_ascii(digest_stream));
434 /* compute MD5 or SHA1 hash */
436 char md[CRYPTO_DIGEST_MAX_SIZE];
441 if (digest_file(jcr, ff_pkt, digest) != 0) {
444 } else if (crypto_digest_finalize(digest, (uint8_t *)md, &size)) {
446 const char *digest_name;
448 digest_buf = (char *)malloc(BASE64_SIZE(size));
449 digest_name = crypto_digest_name(digest);
451 bin_to_base64(digest_buf, BASE64_SIZE(size), md, size, true);
453 if (strcmp(digest_buf, elt.chksum)) {
454 Dmsg4(dbglvl,"%s %s chksum diff. Cat: %s File: %s\n",
464 crypto_digest_free(digest);
477 /* In Incr/Diff accurate mode, we mark all files as seen
478 * When in Full+Base mode, we mark only if the file match exactly
480 if (jcr->getJobLevel() == L_FULL) {
482 /* compute space saved with basefile */
483 jcr->base_size += ff_pkt->statp.st_size;
484 accurate_mark_file_as_seen(jcr, &elt);
487 accurate_mark_file_as_seen(jcr, &elt);
491 unstrip_path(ff_pkt);
496 * TODO: use big buffer from htable
498 int accurate_cmd(JCR *jcr)
500 BSOCK *dir = jcr->dir_bsock;
501 int lstat_pos, chksum_pos;
505 if (job_canceled(jcr)) {
508 if (sscanf(dir->msg, "accurate files=%ld", &nb) != 1) {
509 dir->fsend(_("2991 Bad accurate command\n"));
513 jcr->accurate = true;
515 accurate_init(jcr, nb);
518 * buffer = sizeof(CurFile) + dirmsg
519 * dirmsg = fname + \0 + lstat + \0 + checksum + \0 + delta_seq + \0
521 /* get current files */
522 while (dir->recv() >= 0) {
523 lstat_pos = strlen(dir->msg) + 1;
524 if (lstat_pos < dir->msglen) {
525 chksum_pos = lstat_pos + strlen(dir->msg + lstat_pos) + 1;
527 if (chksum_pos >= dir->msglen) {
528 chksum_pos = lstat_pos - 1; /* tweak: no checksum, point to the last \0 */
531 delta_seq = str_to_int32(dir->msg +
533 strlen(dir->msg + chksum_pos) + 1);
536 accurate_add_file(jcr, dir->msglen,
538 dir->msg + lstat_pos, /* LStat */
539 dir->msg + chksum_pos, /* CheckSum */
540 delta_seq); /* Delta Sequence */
545 extern void *start_heap;
547 char b1[50], b2[50], b3[50], b4[50], b5[50];
548 Dmsg5(dbglvl," Heap: heap=%s smbytes=%s max_bytes=%s bufs=%s max_bufs=%s\n",
549 edit_uint64_with_commas((char *)sbrk(0)-(char *)start_heap, b1),
550 edit_uint64_with_commas(sm_bytes, b2),
551 edit_uint64_with_commas(sm_max_bytes, b3),
552 edit_uint64_with_commas(sm_buffers, b4),
553 edit_uint64_with_commas(sm_max_buffers, b5));