2 Bacula® - The Network Backup Solution
4 Copyright (C) 2000-2011 Free Software Foundation Europe e.V.
6 The main author of Bacula is Kern Sibbald, with contributions from
7 many others, a complete list can be found in the file AUTHORS.
8 This program is Free Software; you can redistribute it and/or
9 modify it under the terms of version three of the GNU Affero General Public
10 License as published by the Free Software Foundation and included
13 This program is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 General Public License for more details.
18 You should have received a copy of the GNU Affero General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
23 Bacula® is a registered trademark of Kern Sibbald.
24 The licensor of Bacula is the Free Software Foundation Europe
25 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
26 Switzerland, email:ftf@fsfeurope.org.
36 static int dbglvl=100;
38 typedef struct PrivateCurFile {
47 bool accurate_mark_file_as_seen(JCR *jcr, char *fname)
49 if (!jcr->accurate || !jcr->file_list) {
52 /* TODO: just use elt->seen = 1 */
53 CurFile *temp = (CurFile *)jcr->file_list->lookup(fname);
55 temp->seen = 1; /* records are in memory */
56 Dmsg1(dbglvl, "marked <%s> as seen\n", fname);
58 Dmsg1(dbglvl, "<%s> not found to be marked as seen\n", fname);
63 static bool accurate_mark_file_as_seen(JCR *jcr, CurFile *elt)
65 /* TODO: just use elt->seen = 1 */
66 CurFile *temp = (CurFile *)jcr->file_list->lookup(elt->fname);
68 temp->seen = 1; /* records are in memory */
73 static bool accurate_lookup(JCR *jcr, char *fname, CurFile *ret)
78 CurFile *temp = (CurFile *)jcr->file_list->lookup(fname);
80 memcpy(ret, temp, sizeof(CurFile));
82 Dmsg1(dbglvl, "lookup <%s> ok\n", fname);
88 static bool accurate_init(JCR *jcr, int nbfile)
91 jcr->file_list = (htable *)malloc(sizeof(htable));
92 jcr->file_list->init(elt, &elt->link, nbfile);
96 static bool accurate_send_base_file_list(JCR *jcr)
102 int stream = STREAM_UNIX_ATTRIBUTES;
104 if (!jcr->accurate || jcr->getJobLevel() != L_FULL) {
108 if (jcr->file_list == NULL) {
112 ff_pkt = init_find_files();
113 ff_pkt->type = FT_BASE;
115 foreach_htable(elt, jcr->file_list) {
117 Dmsg2(dbglvl, "base file fname=%s seen=%i\n", elt->fname, elt->seen);
118 /* TODO: skip the decode and use directly the lstat field */
119 decode_stat(elt->lstat, &statc, sizeof(statc), &LinkFIc); /* decode catalog stat */
120 ff_pkt->fname = elt->fname;
121 ff_pkt->statp = statc;
122 encode_and_send_attributes(jcr, ff_pkt, stream);
127 term_find_files(ff_pkt);
132 /* This function is called at the end of backup
133 * We walk over all hash disk element, and we check
136 static bool accurate_send_deleted_list(JCR *jcr)
142 int stream = STREAM_UNIX_ATTRIBUTES;
144 if (!jcr->accurate) {
148 if (jcr->file_list == NULL) {
152 ff_pkt = init_find_files();
153 ff_pkt->type = FT_DELETED;
155 foreach_htable(elt, jcr->file_list) {
156 if (elt->seen || plugin_check_file(jcr, elt->fname)) {
159 Dmsg2(dbglvl, "deleted fname=%s seen=%i\n", elt->fname, elt->seen);
160 /* TODO: skip the decode and use directly the lstat field */
161 decode_stat(elt->lstat, &statc, sizeof(statc), &LinkFIc); /* decode catalog stat */
162 ff_pkt->fname = elt->fname;
163 ff_pkt->statp.st_mtime = statc.st_mtime;
164 ff_pkt->statp.st_ctime = statc.st_ctime;
165 encode_and_send_attributes(jcr, ff_pkt, stream);
169 term_find_files(ff_pkt);
173 void accurate_free(JCR *jcr)
175 if (jcr->file_list) {
176 jcr->file_list->destroy();
177 free(jcr->file_list);
178 jcr->file_list = NULL;
182 /* Send the deleted or the base file list and cleanup */
183 bool accurate_finish(JCR *jcr)
187 if (jcr->is_canceled() || jcr->is_incomplete()) {
192 if (jcr->is_JobLevel(L_FULL)) {
193 if (!jcr->rerunning) {
194 ret = accurate_send_base_file_list(jcr);
197 ret = accurate_send_deleted_list(jcr);
200 if (jcr->is_JobLevel(L_FULL)) {
201 Jmsg(jcr, M_INFO, 0, _("Space saved with Base jobs: %lld MB\n"),
202 jcr->base_size/(1024*1024));
208 static bool accurate_add_file(JCR *jcr, uint32_t len,
209 char *fname, char *lstat, char *chksum,
215 /* we store CurFile, fname and ctime/mtime in the same chunk
216 * we need one extra byte to handle an empty chksum
218 item = (CurFile *)jcr->file_list->hash_malloc(sizeof(CurFile)+len+3);
221 /* TODO: see if we can optimize this part with memcpy instead of strcpy */
222 item->fname = (char *)item+sizeof(CurFile);
223 strcpy(item->fname, fname);
225 item->lstat = item->fname+strlen(item->fname)+1;
226 strcpy(item->lstat, lstat);
228 item->chksum = item->lstat+strlen(item->lstat)+1;
229 strcpy(item->chksum, chksum);
231 item->delta_seq = delta;
233 jcr->file_list->insert(item->fname, item);
235 Dmsg4(dbglvl, "add fname=<%s> lstat=%s delta_seq=%i chksum=%s\n",
236 fname, lstat, delta, chksum);
241 * This function is called for each file seen in fileset.
242 * We check in file_list hash if fname have been backuped
243 * the last time. After we can compare Lstat field.
244 * Full Lstat usage have been removed on 6612
246 * Returns: true if file has changed (must be backed up)
247 * false file not changed
249 bool accurate_check_file(JCR *jcr, FF_PKT *ff_pkt)
251 int digest_stream = STREAM_NONE;
252 DIGEST *digest = NULL;
261 ff_pkt->delta_seq = 0;
262 ff_pkt->accurate_found = false;
264 if (!jcr->accurate && !jcr->rerunning) {
268 if (!jcr->file_list) {
269 return true; /* Not initialized properly */
274 if (S_ISDIR(ff_pkt->statp.st_mode)) {
275 fname = ff_pkt->link;
277 fname = ff_pkt->fname;
280 if (!accurate_lookup(jcr, fname, &elt)) {
281 Dmsg1(dbglvl, "accurate %s (not found)\n", fname);
286 ff_pkt->accurate_found = true;
287 ff_pkt->delta_seq = elt.delta_seq;
289 decode_stat(elt.lstat, &statc, sizeof(statc), &LinkFIc); /* decode catalog stat */
291 if (!jcr->rerunning && (jcr->getJobLevel() == L_FULL)) {
292 opts = ff_pkt->BaseJobOpts;
294 opts = ff_pkt->AccurateOpts;
298 * Loop over options supplied by user and verify the
299 * fields he requests.
301 for (char *p=opts; !stat && *p; p++) {
302 char ed1[30], ed2[30];
304 case 'i': /* compare INODEs */
305 if (statc.st_ino != ff_pkt->statp.st_ino) {
306 Dmsg3(dbglvl-1, "%s st_ino differ. Cat: %s File: %s\n",
308 edit_uint64((uint64_t)statc.st_ino, ed1),
309 edit_uint64((uint64_t)ff_pkt->statp.st_ino, ed2));
313 case 'p': /* permissions bits */
314 /* TODO: If something change only in perm, user, group
315 * Backup only the attribute stream
317 if (statc.st_mode != ff_pkt->statp.st_mode) {
318 Dmsg3(dbglvl-1, "%s st_mode differ. Cat: %x File: %x\n",
320 (uint32_t)statc.st_mode, (uint32_t)ff_pkt->statp.st_mode);
324 case 'n': /* number of links */
325 if (statc.st_nlink != ff_pkt->statp.st_nlink) {
326 Dmsg3(dbglvl-1, "%s st_nlink differ. Cat: %d File: %d\n",
328 (uint32_t)statc.st_nlink, (uint32_t)ff_pkt->statp.st_nlink);
332 case 'u': /* user id */
333 if (statc.st_uid != ff_pkt->statp.st_uid) {
334 Dmsg3(dbglvl-1, "%s st_uid differ. Cat: %u File: %u\n",
336 (uint32_t)statc.st_uid, (uint32_t)ff_pkt->statp.st_uid);
340 case 'g': /* group id */
341 if (statc.st_gid != ff_pkt->statp.st_gid) {
342 Dmsg3(dbglvl-1, "%s st_gid differ. Cat: %u File: %u\n",
344 (uint32_t)statc.st_gid, (uint32_t)ff_pkt->statp.st_gid);
349 if (statc.st_size != ff_pkt->statp.st_size) {
350 Dmsg3(dbglvl-1, "%s st_size differ. Cat: %s File: %s\n",
352 edit_uint64((uint64_t)statc.st_size, ed1),
353 edit_uint64((uint64_t)ff_pkt->statp.st_size, ed2));
357 case 'a': /* access time */
358 if (statc.st_atime != ff_pkt->statp.st_atime) {
359 Dmsg1(dbglvl-1, "%s st_atime differs\n", fname);
363 case 'm': /* modification time */
364 if (statc.st_mtime != ff_pkt->statp.st_mtime) {
365 Dmsg1(dbglvl-1, "%s st_mtime differs\n", fname);
369 case 'c': /* ctime */
370 if (statc.st_ctime != ff_pkt->statp.st_ctime) {
371 Dmsg1(dbglvl-1, "%s st_ctime differs\n", fname);
375 case 'd': /* file size decrease */
376 if (statc.st_size > ff_pkt->statp.st_size) {
377 Dmsg3(dbglvl-1, "%s st_size decrease. Cat: %s File: %s\n",
379 edit_uint64((uint64_t)statc.st_size, ed1),
380 edit_uint64((uint64_t)ff_pkt->statp.st_size, ed2));
384 case 'A': /* Always backup a file */
387 /* TODO: cleanup and factorise this function with verify.c */
388 case '5': /* compare MD5 */
389 case '1': /* compare SHA1 */
391 * The remainder of the function is all about getting the checksum.
392 * First we initialise, then we read files, other streams and Finder Info.
394 if (!stat && ff_pkt->type != FT_LNKSAVED &&
395 (S_ISREG(ff_pkt->statp.st_mode) &&
396 ff_pkt->flags & (FO_MD5|FO_SHA1|FO_SHA256|FO_SHA512)))
399 if (!*elt.chksum && !jcr->rerunning) {
400 Jmsg(jcr, M_WARNING, 0, _("Cannot verify checksum for %s\n"),
407 * Create our digest context. If this fails, the digest will be set
408 * to NULL and not used.
410 if (ff_pkt->flags & FO_MD5) {
411 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_MD5);
412 digest_stream = STREAM_MD5_DIGEST;
414 } else if (ff_pkt->flags & FO_SHA1) {
415 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA1);
416 digest_stream = STREAM_SHA1_DIGEST;
418 } else if (ff_pkt->flags & FO_SHA256) {
419 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA256);
420 digest_stream = STREAM_SHA256_DIGEST;
422 } else if (ff_pkt->flags & FO_SHA512) {
423 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA512);
424 digest_stream = STREAM_SHA512_DIGEST;
427 /* Did digest initialization fail? */
428 if (digest_stream != STREAM_NONE && digest == NULL) {
429 Jmsg(jcr, M_WARNING, 0, _("%s digest initialization failed\n"),
430 stream_to_ascii(digest_stream));
433 /* compute MD5 or SHA1 hash */
435 char md[CRYPTO_DIGEST_MAX_SIZE];
440 if (digest_file(jcr, ff_pkt, digest) != 0) {
443 } else if (crypto_digest_finalize(digest, (uint8_t *)md, &size)) {
445 const char *digest_name;
447 digest_buf = (char *)malloc(BASE64_SIZE(size));
448 digest_name = crypto_digest_name(digest);
450 bin_to_base64(digest_buf, BASE64_SIZE(size), md, size, true);
452 if (strcmp(digest_buf, elt.chksum)) {
453 Dmsg4(dbglvl,"%s %s chksum diff. Cat: %s File: %s\n",
463 crypto_digest_free(digest);
476 /* In Incr/Diff accurate mode, we mark all files as seen
477 * When in Full+Base mode, we mark only if the file match exactly
479 if (jcr->getJobLevel() == L_FULL) {
481 /* compute space saved with basefile */
482 jcr->base_size += ff_pkt->statp.st_size;
483 accurate_mark_file_as_seen(jcr, &elt);
486 accurate_mark_file_as_seen(jcr, &elt);
490 unstrip_path(ff_pkt);
495 * TODO: use big buffer from htable
497 int accurate_cmd(JCR *jcr)
499 BSOCK *dir = jcr->dir_bsock;
500 int lstat_pos, chksum_pos;
504 if (job_canceled(jcr)) {
507 if (sscanf(dir->msg, "accurate files=%ld", &nb) != 1) {
508 dir->fsend(_("2991 Bad accurate command\n"));
512 jcr->accurate = true;
514 accurate_init(jcr, nb);
517 * buffer = sizeof(CurFile) + dirmsg
518 * dirmsg = fname + \0 + lstat + \0 + checksum + \0 + delta_seq + \0
520 /* get current files */
521 while (dir->recv() >= 0) {
522 lstat_pos = strlen(dir->msg) + 1;
523 if (lstat_pos < dir->msglen) {
524 chksum_pos = lstat_pos + strlen(dir->msg + lstat_pos) + 1;
526 if (chksum_pos >= dir->msglen) {
527 chksum_pos = lstat_pos - 1; /* tweak: no checksum, point to the last \0 */
530 delta_seq = str_to_int32(dir->msg +
532 strlen(dir->msg + chksum_pos) + 1);
535 accurate_add_file(jcr, dir->msglen,
537 dir->msg + lstat_pos, /* LStat */
538 dir->msg + chksum_pos, /* CheckSum */
539 delta_seq); /* Delta Sequence */
544 extern void *start_heap;
546 char b1[50], b2[50], b3[50], b4[50], b5[50];
547 Dmsg5(dbglvl," Heap: heap=%s smbytes=%s max_bytes=%s bufs=%s max_bufs=%s\n",
548 edit_uint64_with_commas((char *)sbrk(0)-(char *)start_heap, b1),
549 edit_uint64_with_commas(sm_bytes, b2),
550 edit_uint64_with_commas(sm_max_bytes, b3),
551 edit_uint64_with_commas(sm_buffers, b4),
552 edit_uint64_with_commas(sm_max_buffers, b5));