2 Bacula® - The Network Backup Solution
4 Copyright (C) 2000-2009 Free Software Foundation Europe e.V.
6 The main author of Bacula is Kern Sibbald, with contributions from
7 many others, a complete list can be found in the file AUTHORS.
8 This program is Free Software; you can redistribute it and/or
9 modify it under the terms of version three of the GNU Affero General Public
10 License as published by the Free Software Foundation and included
13 This program is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 General Public License for more details.
18 You should have received a copy of the GNU Affero General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
23 Bacula® is a registered trademark of Kern Sibbald.
24 The licensor of Bacula is the Free Software Foundation Europe
25 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
26 Switzerland, email:ftf@fsfeurope.org.
36 static int dbglvl=100;
38 typedef struct PrivateCurFile {
47 bool accurate_mark_file_as_seen(JCR *jcr, char *fname)
49 if (!jcr->accurate || !jcr->file_list) {
52 /* TODO: just use elt->seen = 1 */
53 CurFile *temp = (CurFile *)jcr->file_list->lookup(fname);
55 temp->seen = 1; /* records are in memory */
56 Dmsg1(dbglvl, "marked <%s> as seen\n", fname);
58 Dmsg1(dbglvl, "<%s> not found to be marked as seen\n", fname);
63 static bool accurate_mark_file_as_seen(JCR *jcr, CurFile *elt)
65 /* TODO: just use elt->seen = 1 */
66 CurFile *temp = (CurFile *)jcr->file_list->lookup(elt->fname);
68 temp->seen = 1; /* records are in memory */
73 static bool accurate_lookup(JCR *jcr, char *fname, CurFile *ret)
78 CurFile *temp = (CurFile *)jcr->file_list->lookup(fname);
80 memcpy(ret, temp, sizeof(CurFile));
82 Dmsg1(dbglvl, "lookup <%s> ok\n", fname);
88 static bool accurate_init(JCR *jcr, int nbfile)
91 jcr->file_list = (htable *)malloc(sizeof(htable));
92 jcr->file_list->init(elt, &elt->link, nbfile);
96 static bool accurate_send_base_file_list(JCR *jcr)
102 int stream = STREAM_UNIX_ATTRIBUTES;
104 if (!jcr->accurate || jcr->getJobLevel() != L_FULL) {
108 if (jcr->file_list == NULL) {
112 ff_pkt = init_find_files();
113 ff_pkt->type = FT_BASE;
115 foreach_htable(elt, jcr->file_list) {
117 Dmsg2(dbglvl, "base file fname=%s seen=%i\n", elt->fname, elt->seen);
118 /* TODO: skip the decode and use directly the lstat field */
119 decode_stat(elt->lstat, &statc, &LinkFIc); /* decode catalog stat */
120 ff_pkt->fname = elt->fname;
121 ff_pkt->statp = statc;
122 encode_and_send_attributes(jcr, ff_pkt, stream);
127 term_find_files(ff_pkt);
132 /* This function is called at the end of backup
133 * We walk over all hash disk element, and we check
136 static bool accurate_send_deleted_list(JCR *jcr)
142 int stream = STREAM_UNIX_ATTRIBUTES;
144 if (!jcr->accurate) {
148 if (jcr->file_list == NULL) {
152 ff_pkt = init_find_files();
153 ff_pkt->type = FT_DELETED;
155 foreach_htable(elt, jcr->file_list) {
156 if (elt->seen || plugin_check_file(jcr, elt->fname)) {
159 Dmsg2(dbglvl, "deleted fname=%s seen=%i\n", elt->fname, elt->seen);
160 /* TODO: skip the decode and use directly the lstat field */
161 decode_stat(elt->lstat, &statc, &LinkFIc); /* decode catalog stat */
162 ff_pkt->fname = elt->fname;
163 ff_pkt->statp.st_mtime = statc.st_mtime;
164 ff_pkt->statp.st_ctime = statc.st_ctime;
165 encode_and_send_attributes(jcr, ff_pkt, stream);
169 term_find_files(ff_pkt);
173 void accurate_free(JCR *jcr)
175 if (jcr->file_list) {
176 jcr->file_list->destroy();
177 free(jcr->file_list);
178 jcr->file_list = NULL;
182 /* Send the deleted or the base file list and cleanup */
183 bool accurate_finish(JCR *jcr)
187 if (jcr->getJobLevel() == L_FULL) {
188 ret = accurate_send_base_file_list(jcr);
190 ret = accurate_send_deleted_list(jcr);
194 if (jcr->getJobLevel() == L_FULL) {
195 Jmsg(jcr, M_INFO, 0, _("Space saved with Base jobs: %lld MB\n"),
196 jcr->base_size/(1024*1024));
202 static bool accurate_add_file(JCR *jcr, uint32_t len,
203 char *fname, char *lstat, char *chksum,
209 /* we store CurFile, fname and ctime/mtime in the same chunk */
210 item = (CurFile *)jcr->file_list->hash_malloc(sizeof(CurFile)+len);
213 /* TODO: see if we can optimize this part with memcpy instead of strcpy */
214 item->fname = (char *)item+sizeof(CurFile);
215 strcpy(item->fname, fname);
217 item->lstat = item->fname+strlen(item->fname)+1;
218 strcpy(item->lstat, lstat);
220 item->chksum = item->lstat+strlen(item->lstat)+1;
221 strcpy(item->chksum, chksum);
223 item->delta_seq = delta;
225 jcr->file_list->insert(item->fname, item);
227 Dmsg4(dbglvl, "add fname=<%s> lstat=%s delta_seq=%i chksum=%s\n",
228 fname, lstat, delta, chksum);
233 * This function is called for each file seen in fileset.
234 * We check in file_list hash if fname have been backuped
235 * the last time. After we can compare Lstat field.
236 * Full Lstat usage have been removed on 6612
238 * Returns: true if file has changed (must be backed up)
239 * false file not changed
241 bool accurate_check_file(JCR *jcr, FF_PKT *ff_pkt)
243 int digest_stream = STREAM_NONE;
244 DIGEST *digest = NULL;
253 ff_pkt->delta_seq = 0;
255 if (!jcr->accurate) {
261 if (S_ISDIR(ff_pkt->statp.st_mode)) {
262 fname = ff_pkt->link;
264 fname = ff_pkt->fname;
267 if (!accurate_lookup(jcr, fname, &elt)) {
268 Dmsg1(dbglvl, "accurate %s (not found)\n", fname);
273 ff_pkt->delta_seq = elt.delta_seq;
275 if (elt.seen) { /* file has been seen ? */
276 Dmsg1(dbglvl, "accurate %s (already seen)\n", fname);
280 decode_stat(elt.lstat, &statc, &LinkFIc); /* decode catalog stat */
282 if (jcr->getJobLevel() == L_FULL) {
283 opts = ff_pkt->BaseJobOpts;
285 opts = ff_pkt->AccurateOpts;
289 * Loop over options supplied by user and verify the
290 * fields he requests.
292 for (char *p=opts; !stat && *p; p++) {
293 char ed1[30], ed2[30];
295 case 'i': /* compare INODEs */
296 if (statc.st_ino != ff_pkt->statp.st_ino) {
297 Dmsg3(dbglvl-1, "%s st_ino differ. Cat: %s File: %s\n",
299 edit_uint64((uint64_t)statc.st_ino, ed1),
300 edit_uint64((uint64_t)ff_pkt->statp.st_ino, ed2));
304 case 'p': /* permissions bits */
305 /* TODO: If something change only in perm, user, group
306 * Backup only the attribute stream
308 if (statc.st_mode != ff_pkt->statp.st_mode) {
309 Dmsg3(dbglvl-1, "%s st_mode differ. Cat: %x File: %x\n",
311 (uint32_t)statc.st_mode, (uint32_t)ff_pkt->statp.st_mode);
315 case 'n': /* number of links */
316 if (statc.st_nlink != ff_pkt->statp.st_nlink) {
317 Dmsg3(dbglvl-1, "%s st_nlink differ. Cat: %d File: %d\n",
319 (uint32_t)statc.st_nlink, (uint32_t)ff_pkt->statp.st_nlink);
323 case 'u': /* user id */
324 if (statc.st_uid != ff_pkt->statp.st_uid) {
325 Dmsg3(dbglvl-1, "%s st_uid differ. Cat: %u File: %u\n",
327 (uint32_t)statc.st_uid, (uint32_t)ff_pkt->statp.st_uid);
331 case 'g': /* group id */
332 if (statc.st_gid != ff_pkt->statp.st_gid) {
333 Dmsg3(dbglvl-1, "%s st_gid differ. Cat: %u File: %u\n",
335 (uint32_t)statc.st_gid, (uint32_t)ff_pkt->statp.st_gid);
340 if (statc.st_size != ff_pkt->statp.st_size) {
341 Dmsg3(dbglvl-1, "%s st_size differ. Cat: %s File: %s\n",
343 edit_uint64((uint64_t)statc.st_size, ed1),
344 edit_uint64((uint64_t)ff_pkt->statp.st_size, ed2));
348 case 'a': /* access time */
349 if (statc.st_atime != ff_pkt->statp.st_atime) {
350 Dmsg1(dbglvl-1, "%s st_atime differs\n", fname);
354 case 'm': /* modification time */
355 if (statc.st_mtime != ff_pkt->statp.st_mtime) {
356 Dmsg1(dbglvl-1, "%s st_mtime differs\n", fname);
360 case 'c': /* ctime */
361 if (statc.st_ctime != ff_pkt->statp.st_ctime) {
362 Dmsg1(dbglvl-1, "%s st_ctime differs\n", fname);
366 case 'd': /* file size decrease */
367 if (statc.st_size > ff_pkt->statp.st_size) {
368 Dmsg3(dbglvl-1, "%s st_size decrease. Cat: %s File: %s\n",
370 edit_uint64((uint64_t)statc.st_size, ed1),
371 edit_uint64((uint64_t)ff_pkt->statp.st_size, ed2));
376 /* TODO: cleanup and factorise this function with verify.c */
377 case '5': /* compare MD5 */
378 case '1': /* compare SHA1 */
380 * The remainder of the function is all about getting the checksum.
381 * First we initialise, then we read files, other streams and Finder Info.
383 if (!stat && ff_pkt->type != FT_LNKSAVED &&
384 (S_ISREG(ff_pkt->statp.st_mode) &&
385 ff_pkt->flags & (FO_MD5|FO_SHA1|FO_SHA256|FO_SHA512)))
389 Jmsg(jcr, M_WARNING, 0, _("Can't verify checksum for %s\n"),
396 * Create our digest context. If this fails, the digest will be set
397 * to NULL and not used.
399 if (ff_pkt->flags & FO_MD5) {
400 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_MD5);
401 digest_stream = STREAM_MD5_DIGEST;
403 } else if (ff_pkt->flags & FO_SHA1) {
404 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA1);
405 digest_stream = STREAM_SHA1_DIGEST;
407 } else if (ff_pkt->flags & FO_SHA256) {
408 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA256);
409 digest_stream = STREAM_SHA256_DIGEST;
411 } else if (ff_pkt->flags & FO_SHA512) {
412 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA512);
413 digest_stream = STREAM_SHA512_DIGEST;
416 /* Did digest initialization fail? */
417 if (digest_stream != STREAM_NONE && digest == NULL) {
418 Jmsg(jcr, M_WARNING, 0, _("%s digest initialization failed\n"),
419 stream_to_ascii(digest_stream));
422 /* compute MD5 or SHA1 hash */
424 char md[CRYPTO_DIGEST_MAX_SIZE];
429 if (digest_file(jcr, ff_pkt, digest) != 0) {
432 } else if (crypto_digest_finalize(digest, (uint8_t *)md, &size)) {
434 const char *digest_name;
436 digest_buf = (char *)malloc(BASE64_SIZE(size));
437 digest_name = crypto_digest_name(digest);
439 bin_to_base64(digest_buf, BASE64_SIZE(size), md, size, true);
441 if (strcmp(digest_buf, elt.chksum)) {
442 Dmsg3(dbglvl-1, "%s chksum diff. Cat: %s File: %s\n",
451 crypto_digest_free(digest);
464 /* In Incr/Diff accurate mode, we mark all files as seen
465 * When in Full+Base mode, we mark only if the file match exactly
467 if (jcr->getJobLevel() == L_FULL) {
469 /* compute space saved with basefile */
470 jcr->base_size += ff_pkt->statp.st_size;
471 accurate_mark_file_as_seen(jcr, &elt);
474 accurate_mark_file_as_seen(jcr, &elt);
478 unstrip_path(ff_pkt);
483 * TODO: use big buffer from htable
485 int accurate_cmd(JCR *jcr)
487 BSOCK *dir = jcr->dir_bsock;
488 int lstat_pos, chksum_pos;
492 if (job_canceled(jcr)) {
495 if (sscanf(dir->msg, "accurate files=%ld", &nb) != 1) {
496 dir->fsend(_("2991 Bad accurate command\n"));
500 jcr->accurate = true;
502 accurate_init(jcr, nb);
505 * buffer = sizeof(CurFile) + dirmsg
506 * dirmsg = fname + \0 + lstat + \0 + checksum + \0 + delta_seq + \0
508 /* get current files */
509 while (dir->recv() >= 0) {
510 lstat_pos = strlen(dir->msg) + 1;
511 if (lstat_pos < dir->msglen) {
512 chksum_pos = lstat_pos + strlen(dir->msg + lstat_pos) + 1;
514 if (chksum_pos >= dir->msglen) {
515 chksum_pos = lstat_pos - 1; /* tweak: no checksum, point to the last \0 */
518 delta_seq = str_to_int32(dir->msg +
520 strlen(dir->msg + chksum_pos) + 1);
523 accurate_add_file(jcr, dir->msglen,
525 dir->msg + lstat_pos, /* LStat */
526 dir->msg + chksum_pos, /* CheckSum */
527 delta_seq); /* Delta Sequence */
532 extern void *start_heap;
534 char b1[50], b2[50], b3[50], b4[50], b5[50];
535 Dmsg5(dbglvl," Heap: heap=%s smbytes=%s max_bytes=%s bufs=%s max_bufs=%s\n",
536 edit_uint64_with_commas((char *)sbrk(0)-(char *)start_heap, b1),
537 edit_uint64_with_commas(sm_bytes, b2),
538 edit_uint64_with_commas(sm_max_bytes, b3),
539 edit_uint64_with_commas(sm_buffers, b4),
540 edit_uint64_with_commas(sm_max_buffers, b5));