2 Bacula® - The Network Backup Solution
4 Copyright (C) 2000-2011 Free Software Foundation Europe e.V.
6 The main author of Bacula is Kern Sibbald, with contributions from
7 many others, a complete list can be found in the file AUTHORS.
8 This program is Free Software; you can redistribute it and/or
9 modify it under the terms of version three of the GNU Affero General Public
10 License as published by the Free Software Foundation and included
13 This program is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 General Public License for more details.
18 You should have received a copy of the GNU Affero General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
23 Bacula® is a registered trademark of Kern Sibbald.
24 The licensor of Bacula is the Free Software Foundation Europe
25 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
26 Switzerland, email:ftf@fsfeurope.org.
36 static int dbglvl=100;
38 typedef struct PrivateCurFile {
47 bool accurate_mark_file_as_seen(JCR *jcr, char *fname)
49 if (!jcr->accurate || !jcr->file_list) {
52 /* TODO: just use elt->seen = 1 */
53 CurFile *temp = (CurFile *)jcr->file_list->lookup(fname);
55 temp->seen = 1; /* records are in memory */
56 Dmsg1(dbglvl, "marked <%s> as seen\n", fname);
58 Dmsg1(dbglvl, "<%s> not found to be marked as seen\n", fname);
63 static bool accurate_mark_file_as_seen(JCR *jcr, CurFile *elt)
65 /* TODO: just use elt->seen = 1 */
66 CurFile *temp = (CurFile *)jcr->file_list->lookup(elt->fname);
68 temp->seen = 1; /* records are in memory */
73 static bool accurate_lookup(JCR *jcr, char *fname, CurFile *ret)
78 CurFile *temp = (CurFile *)jcr->file_list->lookup(fname);
80 memcpy(ret, temp, sizeof(CurFile));
82 Dmsg1(dbglvl, "lookup <%s> ok\n", fname);
88 static bool accurate_init(JCR *jcr, int nbfile)
91 jcr->file_list = (htable *)malloc(sizeof(htable));
92 jcr->file_list->init(elt, &elt->link, nbfile);
96 static bool accurate_send_base_file_list(JCR *jcr)
102 int stream = STREAM_UNIX_ATTRIBUTES;
104 if (!jcr->accurate || jcr->getJobLevel() != L_FULL) {
108 if (jcr->file_list == NULL) {
112 ff_pkt = init_find_files();
113 ff_pkt->type = FT_BASE;
115 foreach_htable(elt, jcr->file_list) {
117 Dmsg2(dbglvl, "base file fname=%s seen=%i\n", elt->fname, elt->seen);
118 /* TODO: skip the decode and use directly the lstat field */
119 decode_stat(elt->lstat, &statc, sizeof(statc), &LinkFIc); /* decode catalog stat */
120 ff_pkt->fname = elt->fname;
121 ff_pkt->statp = statc;
122 encode_and_send_attributes(jcr, ff_pkt, stream);
127 term_find_files(ff_pkt);
132 /* This function is called at the end of backup
133 * We walk over all hash disk element, and we check
136 static bool accurate_send_deleted_list(JCR *jcr)
142 int stream = STREAM_UNIX_ATTRIBUTES;
144 if (!jcr->accurate) {
148 if (jcr->file_list == NULL) {
152 ff_pkt = init_find_files();
153 ff_pkt->type = FT_DELETED;
155 foreach_htable(elt, jcr->file_list) {
156 if (elt->seen || plugin_check_file(jcr, elt->fname)) {
159 Dmsg2(dbglvl, "deleted fname=%s seen=%i\n", elt->fname, elt->seen);
160 /* TODO: skip the decode and use directly the lstat field */
161 decode_stat(elt->lstat, &statc, sizeof(statc), &LinkFIc); /* decode catalog stat */
162 ff_pkt->fname = elt->fname;
163 ff_pkt->statp.st_mtime = statc.st_mtime;
164 ff_pkt->statp.st_ctime = statc.st_ctime;
165 encode_and_send_attributes(jcr, ff_pkt, stream);
169 term_find_files(ff_pkt);
173 void accurate_free(JCR *jcr)
175 if (jcr->file_list) {
176 jcr->file_list->destroy();
177 free(jcr->file_list);
178 jcr->file_list = NULL;
182 /* Send the deleted or the base file list and cleanup */
183 bool accurate_finish(JCR *jcr)
187 if (jcr->is_canceled() || jcr->is_incomplete()) {
192 if (jcr->is_JobLevel(L_FULL)) {
193 if (!jcr->rerunning) {
194 ret = accurate_send_base_file_list(jcr);
197 ret = accurate_send_deleted_list(jcr);
200 if (jcr->is_JobLevel(L_FULL)) {
201 Jmsg(jcr, M_INFO, 0, _("Space saved with Base jobs: %lld MB\n"),
202 jcr->base_size/(1024*1024));
208 static bool accurate_add_file(JCR *jcr, uint32_t len,
209 char *fname, char *lstat, char *chksum,
215 /* we store CurFile, fname and ctime/mtime in the same chunk
216 * we need some extra bytes to handle an empty chksum
218 item = (CurFile *)jcr->file_list->hash_malloc(sizeof(CurFile)+len+3);
221 /* TODO: see if we can optimize this part with memcpy instead of strcpy */
222 item->fname = (char *)item+sizeof(CurFile);
223 strcpy(item->fname, fname);
225 item->lstat = item->fname+strlen(item->fname)+1;
226 strcpy(item->lstat, lstat);
228 item->chksum = item->lstat+strlen(item->lstat)+1;
229 strcpy(item->chksum, chksum);
231 item->delta_seq = delta;
233 jcr->file_list->insert(item->fname, item);
235 Dmsg4(dbglvl, "add fname=<%s> lstat=%s delta_seq=%i chksum=%s\n",
236 fname, lstat, delta, chksum);
241 * This function is called for each file seen in fileset.
242 * We check in file_list hash if fname have been backuped
243 * the last time. After we can compare Lstat field.
244 * Full Lstat usage have been removed on 6612
246 * Returns: true if file has changed (must be backed up)
247 * false file not changed
249 bool accurate_check_file(JCR *jcr, FF_PKT *ff_pkt)
251 int digest_stream = STREAM_NONE;
252 DIGEST *digest = NULL;
261 ff_pkt->delta_seq = 0;
263 if (!jcr->accurate && !jcr->rerunning) {
269 if (S_ISDIR(ff_pkt->statp.st_mode)) {
270 fname = ff_pkt->link;
272 fname = ff_pkt->fname;
275 if (!accurate_lookup(jcr, fname, &elt)) {
276 Dmsg1(dbglvl, "accurate %s (not found)\n", fname);
281 ff_pkt->delta_seq = elt.delta_seq;
283 if (elt.seen) { /* file has been seen ? */
284 Dmsg1(dbglvl, "accurate %s (already seen)\n", fname);
288 decode_stat(elt.lstat, &statc, sizeof(statc), &LinkFIc); /* decode catalog stat */
290 if (!jcr->rerunning && (jcr->getJobLevel() == L_FULL)) {
291 opts = ff_pkt->BaseJobOpts;
293 opts = ff_pkt->AccurateOpts;
297 * Loop over options supplied by user and verify the
298 * fields he requests.
300 for (char *p=opts; !stat && *p; p++) {
301 char ed1[30], ed2[30];
303 case 'i': /* compare INODEs */
304 if (statc.st_ino != ff_pkt->statp.st_ino) {
305 Dmsg3(dbglvl-1, "%s st_ino differ. Cat: %s File: %s\n",
307 edit_uint64((uint64_t)statc.st_ino, ed1),
308 edit_uint64((uint64_t)ff_pkt->statp.st_ino, ed2));
312 case 'p': /* permissions bits */
313 /* TODO: If something change only in perm, user, group
314 * Backup only the attribute stream
316 if (statc.st_mode != ff_pkt->statp.st_mode) {
317 Dmsg3(dbglvl-1, "%s st_mode differ. Cat: %x File: %x\n",
319 (uint32_t)statc.st_mode, (uint32_t)ff_pkt->statp.st_mode);
323 case 'n': /* number of links */
324 if (statc.st_nlink != ff_pkt->statp.st_nlink) {
325 Dmsg3(dbglvl-1, "%s st_nlink differ. Cat: %d File: %d\n",
327 (uint32_t)statc.st_nlink, (uint32_t)ff_pkt->statp.st_nlink);
331 case 'u': /* user id */
332 if (statc.st_uid != ff_pkt->statp.st_uid) {
333 Dmsg3(dbglvl-1, "%s st_uid differ. Cat: %u File: %u\n",
335 (uint32_t)statc.st_uid, (uint32_t)ff_pkt->statp.st_uid);
339 case 'g': /* group id */
340 if (statc.st_gid != ff_pkt->statp.st_gid) {
341 Dmsg3(dbglvl-1, "%s st_gid differ. Cat: %u File: %u\n",
343 (uint32_t)statc.st_gid, (uint32_t)ff_pkt->statp.st_gid);
348 if (statc.st_size != ff_pkt->statp.st_size) {
349 Dmsg3(dbglvl-1, "%s st_size differ. Cat: %s File: %s\n",
351 edit_uint64((uint64_t)statc.st_size, ed1),
352 edit_uint64((uint64_t)ff_pkt->statp.st_size, ed2));
356 case 'a': /* access time */
357 if (statc.st_atime != ff_pkt->statp.st_atime) {
358 Dmsg1(dbglvl-1, "%s st_atime differs\n", fname);
362 case 'm': /* modification time */
363 if (statc.st_mtime != ff_pkt->statp.st_mtime) {
364 Dmsg1(dbglvl-1, "%s st_mtime differs\n", fname);
368 case 'c': /* ctime */
369 if (statc.st_ctime != ff_pkt->statp.st_ctime) {
370 Dmsg1(dbglvl-1, "%s st_ctime differs\n", fname);
374 case 'd': /* file size decrease */
375 if (statc.st_size > ff_pkt->statp.st_size) {
376 Dmsg3(dbglvl-1, "%s st_size decrease. Cat: %s File: %s\n",
378 edit_uint64((uint64_t)statc.st_size, ed1),
379 edit_uint64((uint64_t)ff_pkt->statp.st_size, ed2));
384 /* TODO: cleanup and factorise this function with verify.c */
385 case '5': /* compare MD5 */
386 case '1': /* compare SHA1 */
388 * The remainder of the function is all about getting the checksum.
389 * First we initialise, then we read files, other streams and Finder Info.
391 if (!stat && ff_pkt->type != FT_LNKSAVED &&
392 (S_ISREG(ff_pkt->statp.st_mode) &&
393 ff_pkt->flags & (FO_MD5|FO_SHA1|FO_SHA256|FO_SHA512)))
396 if (!*elt.chksum && !jcr->rerunning) {
397 Jmsg(jcr, M_WARNING, 0, _("Cannot verify checksum for %s\n"),
404 * Create our digest context. If this fails, the digest will be set
405 * to NULL and not used.
407 if (ff_pkt->flags & FO_MD5) {
408 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_MD5);
409 digest_stream = STREAM_MD5_DIGEST;
411 } else if (ff_pkt->flags & FO_SHA1) {
412 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA1);
413 digest_stream = STREAM_SHA1_DIGEST;
415 } else if (ff_pkt->flags & FO_SHA256) {
416 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA256);
417 digest_stream = STREAM_SHA256_DIGEST;
419 } else if (ff_pkt->flags & FO_SHA512) {
420 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA512);
421 digest_stream = STREAM_SHA512_DIGEST;
424 /* Did digest initialization fail? */
425 if (digest_stream != STREAM_NONE && digest == NULL) {
426 Jmsg(jcr, M_WARNING, 0, _("%s digest initialization failed\n"),
427 stream_to_ascii(digest_stream));
430 /* compute MD5 or SHA1 hash */
432 char md[CRYPTO_DIGEST_MAX_SIZE];
437 if (digest_file(jcr, ff_pkt, digest) != 0) {
440 } else if (crypto_digest_finalize(digest, (uint8_t *)md, &size)) {
442 const char *digest_name;
444 digest_buf = (char *)malloc(BASE64_SIZE(size));
445 digest_name = crypto_digest_name(digest);
447 bin_to_base64(digest_buf, BASE64_SIZE(size), md, size, true);
449 if (strcmp(digest_buf, elt.chksum)) {
450 Dmsg4(dbglvl,"%s %s chksum diff. Cat: %s File: %s\n",
460 crypto_digest_free(digest);
473 /* In Incr/Diff accurate mode, we mark all files as seen
474 * When in Full+Base mode, we mark only if the file match exactly
476 if (jcr->getJobLevel() == L_FULL) {
478 /* compute space saved with basefile */
479 jcr->base_size += ff_pkt->statp.st_size;
480 accurate_mark_file_as_seen(jcr, &elt);
483 accurate_mark_file_as_seen(jcr, &elt);
487 unstrip_path(ff_pkt);
492 * TODO: use big buffer from htable
494 int accurate_cmd(JCR *jcr)
496 BSOCK *dir = jcr->dir_bsock;
497 int lstat_pos, chksum_pos;
501 if (job_canceled(jcr)) {
504 if (sscanf(dir->msg, "accurate files=%ld", &nb) != 1) {
505 dir->fsend(_("2991 Bad accurate command\n"));
509 jcr->accurate = true;
511 accurate_init(jcr, nb);
514 * buffer = sizeof(CurFile) + dirmsg
515 * dirmsg = fname + \0 + lstat + \0 + checksum + \0 + delta_seq + \0
517 /* get current files */
518 while (dir->recv() >= 0) {
519 lstat_pos = strlen(dir->msg) + 1;
520 if (lstat_pos < dir->msglen) {
521 chksum_pos = lstat_pos + strlen(dir->msg + lstat_pos) + 1;
523 if (chksum_pos >= dir->msglen) {
524 chksum_pos = lstat_pos - 1; /* tweak: no checksum, point to the last \0 */
527 delta_seq = str_to_int32(dir->msg +
529 strlen(dir->msg + chksum_pos) + 1);
532 accurate_add_file(jcr, dir->msglen,
534 dir->msg + lstat_pos, /* LStat */
535 dir->msg + chksum_pos, /* CheckSum */
536 delta_seq); /* Delta Sequence */
541 extern void *start_heap;
543 char b1[50], b2[50], b3[50], b4[50], b5[50];
544 Dmsg5(dbglvl," Heap: heap=%s smbytes=%s max_bytes=%s bufs=%s max_bufs=%s\n",
545 edit_uint64_with_commas((char *)sbrk(0)-(char *)start_heap, b1),
546 edit_uint64_with_commas(sm_bytes, b2),
547 edit_uint64_with_commas(sm_max_bytes, b3),
548 edit_uint64_with_commas(sm_buffers, b4),
549 edit_uint64_with_commas(sm_max_buffers, b5));