2 Bacula® - The Network Backup Solution
4 Copyright (C) 2000-2011 Free Software Foundation Europe e.V.
6 The main author of Bacula is Kern Sibbald, with contributions from
7 many others, a complete list can be found in the file AUTHORS.
8 This program is Free Software; you can redistribute it and/or
9 modify it under the terms of version three of the GNU Affero General Public
10 License as published by the Free Software Foundation and included
13 This program is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 General Public License for more details.
18 You should have received a copy of the GNU Affero General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
23 Bacula® is a registered trademark of Kern Sibbald.
24 The licensor of Bacula is the Free Software Foundation Europe
25 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
26 Switzerland, email:ftf@fsfeurope.org.
36 static int dbglvl=100;
38 typedef struct PrivateCurFile {
47 bool accurate_mark_file_as_seen(JCR *jcr, char *fname)
49 if (!jcr->accurate || !jcr->file_list) {
52 /* TODO: just use elt->seen = 1 */
53 CurFile *temp = (CurFile *)jcr->file_list->lookup(fname);
55 temp->seen = 1; /* records are in memory */
56 Dmsg1(dbglvl, "marked <%s> as seen\n", fname);
58 Dmsg1(dbglvl, "<%s> not found to be marked as seen\n", fname);
63 static bool accurate_mark_file_as_seen(JCR *jcr, CurFile *elt)
65 /* TODO: just use elt->seen = 1 */
66 CurFile *temp = (CurFile *)jcr->file_list->lookup(elt->fname);
68 temp->seen = 1; /* records are in memory */
73 static bool accurate_lookup(JCR *jcr, char *fname, CurFile *ret)
78 CurFile *temp = (CurFile *)jcr->file_list->lookup(fname);
80 memcpy(ret, temp, sizeof(CurFile));
82 Dmsg1(dbglvl, "lookup <%s> ok\n", fname);
88 static bool accurate_init(JCR *jcr, int nbfile)
91 jcr->file_list = (htable *)malloc(sizeof(htable));
92 jcr->file_list->init(elt, &elt->link, nbfile);
96 static bool accurate_send_base_file_list(JCR *jcr)
102 int stream = STREAM_UNIX_ATTRIBUTES;
104 if (!jcr->accurate || jcr->getJobLevel() != L_FULL) {
108 if (jcr->file_list == NULL) {
112 ff_pkt = init_find_files();
113 ff_pkt->type = FT_BASE;
115 foreach_htable(elt, jcr->file_list) {
117 Dmsg2(dbglvl, "base file fname=%s seen=%i\n", elt->fname, elt->seen);
118 /* TODO: skip the decode and use directly the lstat field */
119 decode_stat(elt->lstat, &statc, sizeof(statc), &LinkFIc); /* decode catalog stat */
120 ff_pkt->fname = elt->fname;
121 ff_pkt->statp = statc;
122 encode_and_send_attributes(jcr, ff_pkt, stream);
127 term_find_files(ff_pkt);
132 /* This function is called at the end of backup
133 * We walk over all hash disk element, and we check
136 static bool accurate_send_deleted_list(JCR *jcr)
142 int stream = STREAM_UNIX_ATTRIBUTES;
144 if (!jcr->accurate) {
148 if (jcr->file_list == NULL) {
152 ff_pkt = init_find_files();
153 ff_pkt->type = FT_DELETED;
155 foreach_htable(elt, jcr->file_list) {
156 if (elt->seen || plugin_check_file(jcr, elt->fname)) {
159 Dmsg2(dbglvl, "deleted fname=%s seen=%i\n", elt->fname, elt->seen);
160 /* TODO: skip the decode and use directly the lstat field */
161 decode_stat(elt->lstat, &statc, sizeof(statc), &LinkFIc); /* decode catalog stat */
162 ff_pkt->fname = elt->fname;
163 ff_pkt->statp.st_mtime = statc.st_mtime;
164 ff_pkt->statp.st_ctime = statc.st_ctime;
165 encode_and_send_attributes(jcr, ff_pkt, stream);
169 term_find_files(ff_pkt);
173 void accurate_free(JCR *jcr)
175 if (jcr->file_list) {
176 jcr->file_list->destroy();
177 free(jcr->file_list);
178 jcr->file_list = NULL;
182 /* Send the deleted or the base file list and cleanup */
183 bool accurate_finish(JCR *jcr)
187 if (jcr->is_canceled() || jcr->is_incomplete()) {
192 if (jcr->is_JobLevel(L_FULL)) {
193 if (!jcr->rerunning) {
194 ret = accurate_send_base_file_list(jcr);
197 ret = accurate_send_deleted_list(jcr);
200 if (jcr->is_JobLevel(L_FULL)) {
201 Jmsg(jcr, M_INFO, 0, _("Space saved with Base jobs: %lld MB\n"),
202 jcr->base_size/(1024*1024));
208 static bool accurate_add_file(JCR *jcr, uint32_t len,
209 char *fname, char *lstat, char *chksum,
215 /* we store CurFile, fname and ctime/mtime in the same chunk
216 * we need one extra byte to handle an empty chksum
218 item = (CurFile *)jcr->file_list->hash_malloc(sizeof(CurFile)+len+1);
221 /* TODO: see if we can optimize this part with memcpy instead of strcpy */
222 item->fname = (char *)item+sizeof(CurFile);
223 strcpy(item->fname, fname);
225 item->lstat = item->fname+strlen(item->fname)+1;
226 strcpy(item->lstat, lstat);
228 item->chksum = item->lstat+strlen(item->lstat)+1;
229 strcpy(item->chksum, chksum);
231 item->delta_seq = delta;
233 jcr->file_list->insert(item->fname, item);
235 Dmsg4(dbglvl, "add fname=<%s> lstat=%s delta_seq=%i chksum=%s\n",
236 fname, lstat, delta, chksum);
241 * This function is called for each file seen in fileset.
242 * We check in file_list hash if fname have been backuped
243 * the last time. After we can compare Lstat field.
244 * Full Lstat usage have been removed on 6612
246 * Returns: true if file has changed (must be backed up)
247 * false file not changed
249 bool accurate_check_file(JCR *jcr, FF_PKT *ff_pkt)
251 int digest_stream = STREAM_NONE;
252 DIGEST *digest = NULL;
261 ff_pkt->delta_seq = 0;
263 if (!jcr->accurate && !jcr->rerunning) {
269 if (S_ISDIR(ff_pkt->statp.st_mode)) {
270 fname = ff_pkt->link;
272 fname = ff_pkt->fname;
275 if (!accurate_lookup(jcr, fname, &elt)) {
276 Dmsg1(dbglvl, "accurate %s (not found)\n", fname);
281 ff_pkt->delta_seq = elt.delta_seq;
283 if (elt.seen) { /* file has been seen ? */
284 Dmsg1(dbglvl, "accurate %s (already seen)\n", fname);
288 decode_stat(elt.lstat, &statc, sizeof(statc), &LinkFIc); /* decode catalog stat */
290 if (!jcr->rerunning && (jcr->getJobLevel() == L_FULL)) {
291 opts = ff_pkt->BaseJobOpts;
293 opts = ff_pkt->AccurateOpts;
297 * Loop over options supplied by user and verify the
298 * fields he requests.
300 for (char *p=opts; !stat && *p; p++) {
301 char ed1[30], ed2[30];
303 case 'i': /* compare INODEs */
304 if (statc.st_ino != ff_pkt->statp.st_ino) {
305 Dmsg3(dbglvl-1, "%s st_ino differ. Cat: %s File: %s\n",
307 edit_uint64((uint64_t)statc.st_ino, ed1),
308 edit_uint64((uint64_t)ff_pkt->statp.st_ino, ed2));
312 case 'p': /* permissions bits */
313 /* TODO: If something change only in perm, user, group
314 * Backup only the attribute stream
316 if (statc.st_mode != ff_pkt->statp.st_mode) {
317 Dmsg3(dbglvl-1, "%s st_mode differ. Cat: %x File: %x\n",
319 (uint32_t)statc.st_mode, (uint32_t)ff_pkt->statp.st_mode);
323 case 'n': /* number of links */
324 if (statc.st_nlink != ff_pkt->statp.st_nlink) {
325 Dmsg3(dbglvl-1, "%s st_nlink differ. Cat: %d File: %d\n",
327 (uint32_t)statc.st_nlink, (uint32_t)ff_pkt->statp.st_nlink);
331 case 'u': /* user id */
332 if (statc.st_uid != ff_pkt->statp.st_uid) {
333 Dmsg3(dbglvl-1, "%s st_uid differ. Cat: %u File: %u\n",
335 (uint32_t)statc.st_uid, (uint32_t)ff_pkt->statp.st_uid);
339 case 'g': /* group id */
340 if (statc.st_gid != ff_pkt->statp.st_gid) {
341 Dmsg3(dbglvl-1, "%s st_gid differ. Cat: %u File: %u\n",
343 (uint32_t)statc.st_gid, (uint32_t)ff_pkt->statp.st_gid);
348 if (statc.st_size != ff_pkt->statp.st_size) {
349 Dmsg3(dbglvl-1, "%s st_size differ. Cat: %s File: %s\n",
351 edit_uint64((uint64_t)statc.st_size, ed1),
352 edit_uint64((uint64_t)ff_pkt->statp.st_size, ed2));
356 case 'a': /* access time */
357 if (statc.st_atime != ff_pkt->statp.st_atime) {
358 Dmsg1(dbglvl-1, "%s st_atime differs\n", fname);
362 case 'm': /* modification time */
363 if (statc.st_mtime != ff_pkt->statp.st_mtime) {
364 Dmsg1(dbglvl-1, "%s st_mtime differs\n", fname);
368 case 'c': /* ctime */
369 if (statc.st_ctime != ff_pkt->statp.st_ctime) {
370 Dmsg1(dbglvl-1, "%s st_ctime differs\n", fname);
374 case 'd': /* file size decrease */
375 if (statc.st_size > ff_pkt->statp.st_size) {
376 Dmsg3(dbglvl-1, "%s st_size decrease. Cat: %s File: %s\n",
378 edit_uint64((uint64_t)statc.st_size, ed1),
379 edit_uint64((uint64_t)ff_pkt->statp.st_size, ed2));
383 case 'A': /* Always backup a file */
386 /* TODO: cleanup and factorise this function with verify.c */
387 case '5': /* compare MD5 */
388 case '1': /* compare SHA1 */
390 * The remainder of the function is all about getting the checksum.
391 * First we initialise, then we read files, other streams and Finder Info.
393 if (!stat && ff_pkt->type != FT_LNKSAVED &&
394 (S_ISREG(ff_pkt->statp.st_mode) &&
395 ff_pkt->flags & (FO_MD5|FO_SHA1|FO_SHA256|FO_SHA512)))
398 if (!*elt.chksum && !jcr->rerunning) {
399 Jmsg(jcr, M_WARNING, 0, _("Cannot verify checksum for %s\n"),
406 * Create our digest context. If this fails, the digest will be set
407 * to NULL and not used.
409 if (ff_pkt->flags & FO_MD5) {
410 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_MD5);
411 digest_stream = STREAM_MD5_DIGEST;
413 } else if (ff_pkt->flags & FO_SHA1) {
414 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA1);
415 digest_stream = STREAM_SHA1_DIGEST;
417 } else if (ff_pkt->flags & FO_SHA256) {
418 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA256);
419 digest_stream = STREAM_SHA256_DIGEST;
421 } else if (ff_pkt->flags & FO_SHA512) {
422 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA512);
423 digest_stream = STREAM_SHA512_DIGEST;
426 /* Did digest initialization fail? */
427 if (digest_stream != STREAM_NONE && digest == NULL) {
428 Jmsg(jcr, M_WARNING, 0, _("%s digest initialization failed\n"),
429 stream_to_ascii(digest_stream));
432 /* compute MD5 or SHA1 hash */
434 char md[CRYPTO_DIGEST_MAX_SIZE];
439 if (digest_file(jcr, ff_pkt, digest) != 0) {
442 } else if (crypto_digest_finalize(digest, (uint8_t *)md, &size)) {
444 const char *digest_name;
446 digest_buf = (char *)malloc(BASE64_SIZE(size));
447 digest_name = crypto_digest_name(digest);
449 bin_to_base64(digest_buf, BASE64_SIZE(size), md, size, true);
451 if (strcmp(digest_buf, elt.chksum)) {
452 Dmsg4(dbglvl,"%s %s chksum diff. Cat: %s File: %s\n",
462 crypto_digest_free(digest);
475 /* In Incr/Diff accurate mode, we mark all files as seen
476 * When in Full+Base mode, we mark only if the file match exactly
478 if (jcr->getJobLevel() == L_FULL) {
480 /* compute space saved with basefile */
481 jcr->base_size += ff_pkt->statp.st_size;
482 accurate_mark_file_as_seen(jcr, &elt);
485 accurate_mark_file_as_seen(jcr, &elt);
489 unstrip_path(ff_pkt);
494 * TODO: use big buffer from htable
496 int accurate_cmd(JCR *jcr)
498 BSOCK *dir = jcr->dir_bsock;
499 int lstat_pos, chksum_pos;
503 if (job_canceled(jcr)) {
506 if (sscanf(dir->msg, "accurate files=%ld", &nb) != 1) {
507 dir->fsend(_("2991 Bad accurate command\n"));
511 jcr->accurate = true;
513 accurate_init(jcr, nb);
516 * buffer = sizeof(CurFile) + dirmsg
517 * dirmsg = fname + \0 + lstat + \0 + checksum + \0 + delta_seq + \0
519 /* get current files */
520 while (dir->recv() >= 0) {
521 lstat_pos = strlen(dir->msg) + 1;
522 if (lstat_pos < dir->msglen) {
523 chksum_pos = lstat_pos + strlen(dir->msg + lstat_pos) + 1;
525 if (chksum_pos >= dir->msglen) {
526 chksum_pos = lstat_pos - 1; /* tweak: no checksum, point to the last \0 */
529 delta_seq = str_to_int32(dir->msg +
531 strlen(dir->msg + chksum_pos) + 1);
534 accurate_add_file(jcr, dir->msglen,
536 dir->msg + lstat_pos, /* LStat */
537 dir->msg + chksum_pos, /* CheckSum */
538 delta_seq); /* Delta Sequence */
543 extern void *start_heap;
545 char b1[50], b2[50], b3[50], b4[50], b5[50];
546 Dmsg5(dbglvl," Heap: heap=%s smbytes=%s max_bytes=%s bufs=%s max_bufs=%s\n",
547 edit_uint64_with_commas((char *)sbrk(0)-(char *)start_heap, b1),
548 edit_uint64_with_commas(sm_bytes, b2),
549 edit_uint64_with_commas(sm_max_bytes, b3),
550 edit_uint64_with_commas(sm_buffers, b4),
551 edit_uint64_with_commas(sm_max_buffers, b5));