2 Bacula(R) - The Network Backup Solution
4 Copyright (C) 2000-2015 Kern Sibbald
5 Copyright (C) 2000-2015 Free Software Foundation Europe e.V.
7 The original author of Bacula is Kern Sibbald, with contributions
8 from many others, a complete list can be found in the file AUTHORS.
10 You may use this file and others of this release according to the
11 license defined in the LICENSE file, which includes the Affero General
12 Public License, v3.0 ("AGPLv3") and some additional permissions and
13 terms pursuant to its AGPLv3 Section 7.
15 This notice must be preserved when any source code is
16 conveyed and/or propagated.
18 Bacula(R) is a registered trademark of Kern Sibbald.
25 static int dbglvl=100;
27 typedef struct PrivateCurFile {
36 bool accurate_mark_file_as_seen(JCR *jcr, char *fname)
38 if (!jcr->accurate || !jcr->file_list) {
41 /* TODO: just use elt->seen = 1 */
42 CurFile *temp = (CurFile *)jcr->file_list->lookup(fname);
44 temp->seen = 1; /* records are in memory */
45 Dmsg1(dbglvl, "marked <%s> as seen\n", fname);
47 Dmsg1(dbglvl, "<%s> not found to be marked as seen\n", fname);
52 static bool accurate_mark_file_as_seen(JCR *jcr, CurFile *elt)
54 /* TODO: just use elt->seen = 1 */
55 CurFile *temp = (CurFile *)jcr->file_list->lookup(elt->fname);
57 temp->seen = 1; /* records are in memory */
62 static bool accurate_lookup(JCR *jcr, char *fname, CurFile *ret)
67 CurFile *temp = (CurFile *)jcr->file_list->lookup(fname);
69 memcpy(ret, temp, sizeof(CurFile));
71 Dmsg1(dbglvl, "lookup <%s> ok\n", fname);
77 static bool accurate_init(JCR *jcr, int nbfile)
80 jcr->file_list = (htable *)malloc(sizeof(htable));
81 jcr->file_list->init(elt, &elt->link, nbfile);
85 static bool accurate_send_base_file_list(JCR *jcr)
92 memset(&bctx, 0, sizeof(bctx));
94 bctx.data_stream = STREAM_UNIX_ATTRIBUTES;
96 if (!jcr->accurate || jcr->getJobLevel() != L_FULL) {
100 if (jcr->file_list == NULL) {
104 bctx.ff_pkt = init_find_files();
105 bctx.ff_pkt->type = FT_BASE;
107 foreach_htable(elt, jcr->file_list) {
109 Dmsg2(dbglvl, "base file fname=%s seen=%i\n", elt->fname, elt->seen);
110 /* TODO: skip the decode and use directly the lstat field */
111 decode_stat(elt->lstat, &statc, sizeof(statc), &LinkFIc); /* decode catalog stat */
112 bctx.ff_pkt->fname = elt->fname;
113 bctx.ff_pkt->statp = statc;
114 encode_and_send_attributes(bctx);
119 term_find_files(bctx.ff_pkt);
124 /* This function is called at the end of backup
125 * We walk over all hash disk element, and we check
128 static bool accurate_send_deleted_list(JCR *jcr)
135 memset(&bctx, 0, sizeof(bctx));
137 bctx.data_stream = STREAM_UNIX_ATTRIBUTES;
139 if (!jcr->accurate) {
143 if (jcr->file_list == NULL) {
147 bctx.ff_pkt = init_find_files();
148 bctx.ff_pkt->type = FT_DELETED;
150 foreach_htable(elt, jcr->file_list) {
151 if (elt->seen || plugin_check_file(jcr, elt->fname)) {
154 Dmsg2(dbglvl, "deleted fname=%s seen=%i\n", elt->fname, elt->seen);
155 /* TODO: skip the decode and use directly the lstat field */
156 decode_stat(elt->lstat, &statc, sizeof(statc), &LinkFIc); /* decode catalog stat */
157 bctx.ff_pkt->fname = elt->fname;
158 bctx.ff_pkt->statp.st_mtime = statc.st_mtime;
159 bctx.ff_pkt->statp.st_ctime = statc.st_ctime;
160 encode_and_send_attributes(bctx);
164 term_find_files(bctx.ff_pkt);
168 void accurate_free(JCR *jcr)
170 if (jcr->file_list) {
171 jcr->file_list->destroy();
172 free(jcr->file_list);
173 jcr->file_list = NULL;
177 /* Send the deleted or the base file list and cleanup */
178 bool accurate_finish(JCR *jcr)
182 if (jcr->is_canceled() || jcr->is_incomplete()) {
187 if (jcr->is_JobLevel(L_FULL)) {
188 if (!jcr->rerunning) {
189 ret = accurate_send_base_file_list(jcr);
192 ret = accurate_send_deleted_list(jcr);
195 if (jcr->is_JobLevel(L_FULL)) {
196 Jmsg(jcr, M_INFO, 0, _("Space saved with Base jobs: %lld MB\n"),
197 jcr->base_size/(1024*1024));
203 static bool accurate_add_file(JCR *jcr, uint32_t len,
204 char *fname, char *lstat, char *chksum,
210 /* we store CurFile, fname and ctime/mtime in the same chunk
211 * we need one extra byte to handle an empty chksum
213 item = (CurFile *)jcr->file_list->hash_malloc(sizeof(CurFile)+len+3);
216 /* TODO: see if we can optimize this part with memcpy instead of strcpy */
217 item->fname = (char *)item+sizeof(CurFile);
218 strcpy(item->fname, fname);
220 item->lstat = item->fname+strlen(item->fname)+1;
221 strcpy(item->lstat, lstat);
223 item->chksum = item->lstat+strlen(item->lstat)+1;
224 strcpy(item->chksum, chksum);
226 item->delta_seq = delta;
228 jcr->file_list->insert(item->fname, item);
230 Dmsg4(dbglvl, "add fname=<%s> lstat=%s delta_seq=%i chksum=%s\n",
231 fname, lstat, delta, chksum);
236 * This function is called for each file seen in fileset.
237 * We check in file_list hash if fname have been backuped
238 * the last time. After we can compare Lstat field.
239 * Full Lstat usage have been removed on 6612
241 * Returns: true if file has changed (must be backed up)
242 * false file not changed
244 bool accurate_check_file(JCR *jcr, FF_PKT *ff_pkt)
246 int digest_stream = STREAM_NONE;
247 DIGEST *digest = NULL;
256 ff_pkt->delta_seq = 0;
257 ff_pkt->accurate_found = false;
259 if (!jcr->accurate && !jcr->rerunning) {
263 if (!jcr->file_list) {
264 return true; /* Not initialized properly */
269 if (S_ISDIR(ff_pkt->statp.st_mode)) {
270 fname = ff_pkt->link;
272 fname = ff_pkt->fname;
275 if (!accurate_lookup(jcr, fname, &elt)) {
276 Dmsg1(dbglvl, "accurate %s (not found)\n", fname);
278 unstrip_path(ff_pkt);
282 unstrip_path(ff_pkt); /* Get full path back */
283 ff_pkt->accurate_found = true;
284 ff_pkt->delta_seq = elt.delta_seq;
286 decode_stat(elt.lstat, &statc, sizeof(statc), &LinkFIc); /* decode catalog stat */
288 if (!jcr->rerunning && (jcr->getJobLevel() == L_FULL)) {
289 opts = ff_pkt->BaseJobOpts;
291 opts = ff_pkt->AccurateOpts;
295 * Loop over options supplied by user and verify the
296 * fields he requests.
298 for (char *p=opts; !stat && *p; p++) {
299 char ed1[30], ed2[30];
301 case 'i': /* compare INODEs */
302 if (statc.st_ino != ff_pkt->statp.st_ino) {
303 Dmsg3(dbglvl-1, "%s st_ino differ. Cat: %s File: %s\n",
305 edit_uint64((uint64_t)statc.st_ino, ed1),
306 edit_uint64((uint64_t)ff_pkt->statp.st_ino, ed2));
310 case 'p': /* permissions bits */
311 /* TODO: If something change only in perm, user, group
312 * Backup only the attribute stream
314 if (statc.st_mode != ff_pkt->statp.st_mode) {
315 Dmsg3(dbglvl-1, "%s st_mode differ. Cat: %x File: %x\n",
317 (uint32_t)statc.st_mode, (uint32_t)ff_pkt->statp.st_mode);
321 case 'n': /* number of links */
322 if (statc.st_nlink != ff_pkt->statp.st_nlink) {
323 Dmsg3(dbglvl-1, "%s st_nlink differ. Cat: %d File: %d\n",
325 (uint32_t)statc.st_nlink, (uint32_t)ff_pkt->statp.st_nlink);
329 case 'u': /* user id */
330 if (statc.st_uid != ff_pkt->statp.st_uid) {
331 Dmsg3(dbglvl-1, "%s st_uid differ. Cat: %u File: %u\n",
333 (uint32_t)statc.st_uid, (uint32_t)ff_pkt->statp.st_uid);
337 case 'g': /* group id */
338 if (statc.st_gid != ff_pkt->statp.st_gid) {
339 Dmsg3(dbglvl-1, "%s st_gid differ. Cat: %u File: %u\n",
341 (uint32_t)statc.st_gid, (uint32_t)ff_pkt->statp.st_gid);
346 if (statc.st_size != ff_pkt->statp.st_size) {
347 Dmsg3(dbglvl-1, "%s st_size differ. Cat: %s File: %s\n",
349 edit_uint64((uint64_t)statc.st_size, ed1),
350 edit_uint64((uint64_t)ff_pkt->statp.st_size, ed2));
354 case 'a': /* access time */
355 if (statc.st_atime != ff_pkt->statp.st_atime) {
356 Dmsg1(dbglvl-1, "%s st_atime differs\n", fname);
360 case 'm': /* modification time */
361 if (statc.st_mtime != ff_pkt->statp.st_mtime) {
362 Dmsg1(dbglvl-1, "%s st_mtime differs\n", fname);
366 case 'M': /* Look mtime/ctime like normal incremental backup */
367 if (ff_pkt->incremental &&
368 (ff_pkt->statp.st_mtime > ff_pkt->save_time &&
369 ((ff_pkt->flags & FO_MTIMEONLY) ||
370 ff_pkt->statp.st_ctime > ff_pkt->save_time)))
372 Dmsg1(dbglvl-1, "%s mtime/ctime more recent than save_time\n", fname);
376 case 'c': /* ctime */
377 if (statc.st_ctime != ff_pkt->statp.st_ctime) {
378 Dmsg1(dbglvl-1, "%s st_ctime differs\n", fname);
382 case 'd': /* file size decrease */
383 if (statc.st_size > ff_pkt->statp.st_size) {
384 Dmsg3(dbglvl-1, "%s st_size decrease. Cat: %s File: %s\n",
386 edit_uint64((uint64_t)statc.st_size, ed1),
387 edit_uint64((uint64_t)ff_pkt->statp.st_size, ed2));
391 case 'A': /* Always backup a file */
394 /* TODO: cleanup and factorise this function with verify.c */
395 case '5': /* compare MD5 */
396 case '1': /* compare SHA1 */
398 * The remainder of the function is all about getting the checksum.
399 * First we initialise, then we read files, other streams and Finder Info.
401 if (!stat && ff_pkt->type != FT_LNKSAVED &&
402 (S_ISREG(ff_pkt->statp.st_mode) &&
403 ff_pkt->flags & (FO_MD5|FO_SHA1|FO_SHA256|FO_SHA512)))
406 if (!*elt.chksum && !jcr->rerunning) {
407 Jmsg(jcr, M_WARNING, 0, _("Cannot verify checksum for %s\n"),
414 * Create our digest context. If this fails, the digest will be set
415 * to NULL and not used.
417 if (ff_pkt->flags & FO_MD5) {
418 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_MD5);
419 digest_stream = STREAM_MD5_DIGEST;
421 } else if (ff_pkt->flags & FO_SHA1) {
422 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA1);
423 digest_stream = STREAM_SHA1_DIGEST;
425 } else if (ff_pkt->flags & FO_SHA256) {
426 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA256);
427 digest_stream = STREAM_SHA256_DIGEST;
429 } else if (ff_pkt->flags & FO_SHA512) {
430 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA512);
431 digest_stream = STREAM_SHA512_DIGEST;
434 /* Did digest initialization fail? */
435 if (digest_stream != STREAM_NONE && digest == NULL) {
436 Jmsg(jcr, M_WARNING, 0, _("%s digest initialization failed\n"),
437 stream_to_ascii(digest_stream));
440 /* compute MD5 or SHA1 hash */
442 char md[CRYPTO_DIGEST_MAX_SIZE];
447 if (digest_file(jcr, ff_pkt, digest) != 0) {
450 } else if (crypto_digest_finalize(digest, (uint8_t *)md, &size)) {
452 const char *digest_name;
454 digest_buf = (char *)malloc(BASE64_SIZE(size));
455 digest_name = crypto_digest_name(digest);
457 bin_to_base64(digest_buf, BASE64_SIZE(size), md, size, true);
459 if (strcmp(digest_buf, elt.chksum)) {
460 Dmsg4(dbglvl,"%s %s chksum diff. Cat: %s File: %s\n",
470 crypto_digest_free(digest);
483 /* In Incr/Diff accurate mode, we mark all files as seen
484 * When in Full+Base mode, we mark only if the file match exactly
486 if (jcr->getJobLevel() == L_FULL) {
488 /* compute space saved with basefile */
489 jcr->base_size += ff_pkt->statp.st_size;
490 accurate_mark_file_as_seen(jcr, &elt);
493 accurate_mark_file_as_seen(jcr, &elt);
501 * TODO: use big buffer from htable
503 int accurate_cmd(JCR *jcr)
505 BSOCK *dir = jcr->dir_bsock;
506 int lstat_pos, chksum_pos;
510 if (job_canceled(jcr)) {
513 if (sscanf(dir->msg, "accurate files=%ld", &nb) != 1) {
514 dir->fsend(_("2991 Bad accurate command\n"));
518 jcr->accurate = true;
520 accurate_init(jcr, nb);
523 * buffer = sizeof(CurFile) + dirmsg
524 * dirmsg = fname + \0 + lstat + \0 + checksum + \0 + delta_seq + \0
526 /* get current files */
527 while (dir->recv() >= 0) {
528 lstat_pos = strlen(dir->msg) + 1;
529 if (lstat_pos < dir->msglen) {
530 chksum_pos = lstat_pos + strlen(dir->msg + lstat_pos) + 1;
532 if (chksum_pos >= dir->msglen) {
533 chksum_pos = lstat_pos - 1; /* tweak: no checksum, point to the last \0 */
536 delta_seq = str_to_int32(dir->msg +
538 strlen(dir->msg + chksum_pos) + 1);
541 accurate_add_file(jcr, dir->msglen,
543 dir->msg + lstat_pos, /* LStat */
544 dir->msg + chksum_pos, /* CheckSum */
545 delta_seq); /* Delta Sequence */
550 extern void *start_heap;
552 char b1[50], b2[50], b3[50], b4[50], b5[50];
553 Dmsg5(dbglvl," Heap: heap=%s smbytes=%s max_bytes=%s bufs=%s max_bufs=%s\n",
554 edit_uint64_with_commas((char *)sbrk(0)-(char *)start_heap, b1),
555 edit_uint64_with_commas(sm_bytes, b2),
556 edit_uint64_with_commas(sm_max_bytes, b3),
557 edit_uint64_with_commas(sm_buffers, b4),
558 edit_uint64_with_commas(sm_max_buffers, b5));