2 Bacula® - The Network Backup Solution
4 Copyright (C) 2000-2008 Free Software Foundation Europe e.V.
6 The main author of Bacula is Kern Sibbald, with contributions from
7 many others, a complete list can be found in the file AUTHORS.
8 This program is Free Software; you can redistribute it and/or
9 modify it under the terms of version two of the GNU General Public
10 License as published by the Free Software Foundation and included
13 This program is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
23 Bacula® is a registered trademark of John Walker.
24 The licensor of Bacula is the Free Software Foundation Europe
25 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
26 Switzerland, email:ftf@fsfeurope.org.
29 * Bacula File Daemon backup.c send file attributes and data
30 * to the Storage daemon.
32 * Kern Sibbald, March MM
40 #include "lib/htable.h"
42 /* Forward referenced functions */
43 int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level);
44 static void strip_path(FF_PKT *ff_pkt);
45 static void unstrip_path(FF_PKT *ff_pkt);
46 static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, DIGEST *signature_digest);
47 static bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream);
48 static bool read_and_send_acl(JCR *jcr, int acltype, int stream);
49 static bool crypto_session_start(JCR *jcr);
50 static void crypto_session_end(JCR *jcr);
51 static bool crypto_session_send(JCR *jcr, BSOCK *sd);
53 typedef struct CurFile {
61 #define accurate_mark_file_as_seen(elt) ((elt)->seen = 1)
62 #define accurate_file_has_been_seen(elt) ((elt)->seen)
65 * This function is called for each file seen in fileset.
66 * We check in file_list hash if fname have been backuped
67 * the last time. After we can compare Lstat field.
68 * Full Lstat usage have been removed on 6612
70 bool accurate_check_file(JCR *jcr, FF_PKT *ff_pkt)
76 if (!jcr->accurate || jcr->JobLevel == L_FULL) {
82 if (S_ISDIR(ff_pkt->statp.st_mode)) {
85 fname = ff_pkt->fname;
88 elt = (CurFile *)jcr->file_list->lookup(fname);
91 Dmsg1(500, "accurate %s = yes (not found)\n", fname);
96 if (accurate_file_has_been_seen(elt)) {
97 Dmsg1(500, "accurate %s = no (already seen)\n", fname);
101 if (elt->mtime != ff_pkt->statp.st_mtime) {
102 Jmsg(jcr, M_SAVED, 0, _("%s st_mtime differs\n"), fname);
104 } else if (elt->ctime != ff_pkt->statp.st_ctime) {
105 Jmsg(jcr, M_SAVED, 0, _("%s st_ctime differs\n"), fname);
109 accurate_mark_file_as_seen(elt);
110 Dmsg2(500, "accurate %s = %i\n", fname, stat);
113 unstrip_path(ff_pkt);
118 * This function doesn't work very well with smartalloc
119 * TODO: use bigbuffer from htable
121 int accurate_cmd(JCR *jcr)
123 BSOCK *dir = jcr->dir_bsock;
131 if (jcr->accurate==false || job_canceled(jcr) || jcr->JobLevel==L_FULL) {
135 if (sscanf(dir->msg, "accurate files=%ld", &nb) != 1) {
136 dir->fsend(_("2991 Bad accurate command\n"));
140 jcr->file_list = (htable *)malloc(sizeof(htable));
141 jcr->file_list->init(elt, &elt->link, nb);
144 * buffer = sizeof(CurFile) + dirmsg
145 * dirmsg = fname + \0 + lstat
147 /* get current files */
148 while (dir->recv() >= 0) {
149 len = strlen(dir->msg);
150 if ((len+1) < dir->msglen) {
151 // elt = (CurFile *)malloc(sizeof(CurFile));
152 // elt->fname = (char *) malloc(dir->msglen+1);
154 /* we store CurFile, fname and ctime/mtime in the same chunk */
155 elt = (CurFile *)malloc(sizeof(CurFile)+len+1);
156 elt->fname = (char *) elt+sizeof(CurFile);
157 strcpy(elt->fname, dir->msg);
158 lstat = dir->msg + len + 1;
159 decode_stat(lstat, &statp, &LinkFIc); /* decode catalog stat */
160 elt->ctime = statp.st_ctime;
161 elt->mtime = statp.st_mtime;
163 jcr->file_list->insert(elt->fname, elt);
164 Dmsg2(500, "add fname=%s lstat=%s\n", elt->fname, lstat);
167 extern void *start_heap;
169 char b1[50], b2[50], b3[50], b4[50], b5[50];
170 Dmsg5(1," Heap: heap=%s smbytes=%s max_bytes=%s bufs=%s max_bufs=%s\n",
171 edit_uint64_with_commas((char *)sbrk(0)-(char *)start_heap, b1),
172 edit_uint64_with_commas(sm_bytes, b2),
173 edit_uint64_with_commas(sm_max_bytes, b3),
174 edit_uint64_with_commas(sm_buffers, b4),
175 edit_uint64_with_commas(sm_max_buffers, b5));
177 // jcr->file_list->stats();
182 bool accurate_send_deleted_list(JCR *jcr)
187 int stream = STREAM_UNIX_ATTRIBUTES;
189 if (jcr->accurate == false || jcr->JobLevel == L_FULL) {
193 if (jcr->file_list == NULL) {
197 ff_pkt = init_find_files();
198 ff_pkt->type = FT_DELETED;
200 foreach_htable (elt, jcr->file_list) {
201 if (!accurate_file_has_been_seen(elt)) { /* already seen */
202 Dmsg2(500, "deleted fname=%s seen=%i\n", elt->fname, elt->seen);
203 ff_pkt->fname = elt->fname;
204 ff_pkt->statp.st_mtime = elt->mtime;
205 ff_pkt->statp.st_ctime = elt->ctime;
206 encode_and_send_attributes(jcr, ff_pkt, stream);
210 term_find_files(ff_pkt);
212 /* TODO: clean htable when this function is not reached ? */
213 if (jcr->file_list) {
214 jcr->file_list->destroy();
215 free(jcr->file_list);
216 jcr->file_list = NULL;
222 * check for BSD nodump flag
224 static bool no_dump(JCR *jcr, FF_PKT *ff_pkt)
226 #if defined(HAVE_CHFLAGS) && defined(UF_NODUMP)
227 if ( (ff_pkt->flags & FO_HONOR_NODUMP) &&
228 (ff_pkt->statp.st_flags & UF_NODUMP) ) {
229 Jmsg(jcr, M_INFO, 1, _(" NODUMP flag set - will not process %s\n"),
231 return true; /* do not backup this file */
234 return false; /* do backup */
238 * Find all the requested files and send them
239 * to the Storage daemon.
241 * Note, we normally carry on a one-way
242 * conversation from this point on with the SD, simply blasting
243 * data to him. To properly know what is going on, we
244 * also run a "heartbeat" monitor which reads the socket and
245 * reacts accordingly (at the moment it has nothing to do
246 * except echo the heartbeat to the Director).
249 bool blast_data_to_storage_daemon(JCR *jcr, char *addr)
253 // TODO landonf: Allow user to specify encryption algorithm
255 sd = jcr->store_bsock;
257 set_jcr_job_status(jcr, JS_Running);
259 Dmsg1(300, "bfiled: opened data connection %d to stored\n", sd->m_fd);
262 CLIENT *client = (CLIENT *)GetNextRes(R_CLIENT, NULL);
266 buf_size = client->max_network_buffer_size;
268 buf_size = 0; /* use default */
270 if (!bnet_set_buffer_size(sd, buf_size, BNET_SETBUF_WRITE)) {
271 set_jcr_job_status(jcr, JS_ErrorTerminated);
272 Jmsg(jcr, M_FATAL, 0, _("Cannot set buffer size FD->SD.\n"));
276 jcr->buf_size = sd->msglen;
277 /* Adjust for compression so that output buffer is
278 * 12 bytes + 0.1% larger than input buffer plus 18 bytes.
279 * This gives a bit extra plus room for the sparse addr if any.
280 * Note, we adjust the read size to be smaller so that the
281 * same output buffer can be used without growing it.
283 * The zlib compression workset is initialized here to minimise
284 * the "per file" load. The jcr member is only set, if the init was successful.
286 jcr->compress_buf_size = jcr->buf_size + ((jcr->buf_size+999) / 1000) + 30;
287 jcr->compress_buf = get_memory(jcr->compress_buf_size);
290 z_stream *pZlibStream = (z_stream*)malloc(sizeof(z_stream));
292 pZlibStream->zalloc = Z_NULL;
293 pZlibStream->zfree = Z_NULL;
294 pZlibStream->opaque = Z_NULL;
295 pZlibStream->state = Z_NULL;
297 if (deflateInit(pZlibStream, Z_DEFAULT_COMPRESSION) == Z_OK) {
298 jcr->pZLIB_compress_workset = pZlibStream;
305 if (!crypto_session_start(jcr)) {
309 set_find_options((FF_PKT *)jcr->ff, jcr->incremental, jcr->mtime);
311 /* in accurate mode, we overwrite the find_one check function */
313 set_find_changed_function((FF_PKT *)jcr->ff, accurate_check_file);
316 start_heartbeat_monitor(jcr);
318 jcr->acl_text = get_pool_memory(PM_MESSAGE);
320 /* Subroutine save_file() is called for each file */
321 if (!find_files(jcr, (FF_PKT *)jcr->ff, save_file, plugin_save)) {
322 ok = false; /* error */
323 set_jcr_job_status(jcr, JS_ErrorTerminated);
326 accurate_send_deleted_list(jcr); /* send deleted list to SD */
328 free_pool_memory(jcr->acl_text);
330 stop_heartbeat_monitor(jcr);
332 sd->signal(BNET_EOD); /* end of sending data */
338 if (jcr->compress_buf) {
339 free_pool_memory(jcr->compress_buf);
340 jcr->compress_buf = NULL;
342 if (jcr->pZLIB_compress_workset) {
343 /* Free the zlib stream */
345 deflateEnd((z_stream *)jcr->pZLIB_compress_workset);
347 free (jcr->pZLIB_compress_workset);
348 jcr->pZLIB_compress_workset = NULL;
350 crypto_session_end(jcr);
353 Dmsg1(100, "end blast_data ok=%d\n", ok);
357 static bool crypto_session_start(JCR *jcr)
359 crypto_cipher_t cipher = CRYPTO_CIPHER_AES_128_CBC;
362 * Create encryption session data and a cached, DER-encoded session data
363 * structure. We use a single session key for each backup, so we'll encode
364 * the session data only once.
366 if (jcr->crypto.pki_encrypt) {
369 /* Create per-job session encryption context */
370 jcr->crypto.pki_session = crypto_session_new(cipher, jcr->crypto.pki_recipients);
372 /* Get the session data size */
373 if (!crypto_session_encode(jcr->crypto.pki_session, (uint8_t *)0, &size)) {
374 Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n"));
378 /* Allocate buffer */
379 jcr->crypto.pki_session_encoded = get_memory(size);
381 /* Encode session data */
382 if (!crypto_session_encode(jcr->crypto.pki_session, (uint8_t *)jcr->crypto.pki_session_encoded, &size)) {
383 Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n"));
387 /* ... and store the encoded size */
388 jcr->crypto.pki_session_encoded_size = size;
390 /* Allocate the encryption/decryption buffer */
391 jcr->crypto.crypto_buf = get_memory(CRYPTO_CIPHER_MAX_BLOCK_SIZE);
396 static void crypto_session_end(JCR *jcr)
398 if (jcr->crypto.crypto_buf) {
399 free_pool_memory(jcr->crypto.crypto_buf);
400 jcr->crypto.crypto_buf = NULL;
402 if (jcr->crypto.pki_session) {
403 crypto_session_free(jcr->crypto.pki_session);
405 if (jcr->crypto.pki_session_encoded) {
406 free_pool_memory(jcr->crypto.pki_session_encoded);
407 jcr->crypto.pki_session_encoded = NULL;
411 static bool crypto_session_send(JCR *jcr, BSOCK *sd)
415 /* Send our header */
416 Dmsg2(100, "Send hdr fi=%ld stream=%d\n", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA);
417 sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA);
420 sd->msg = jcr->crypto.pki_session_encoded;
421 sd->msglen = jcr->crypto.pki_session_encoded_size;
422 jcr->JobBytes += sd->msglen;
424 Dmsg1(100, "Send data len=%d\n", sd->msglen);
427 sd->signal(BNET_EOD);
433 * Called here by find() for each file included.
434 * This is a callback. The original is find_files() above.
436 * Send the file and its data to the Storage daemon.
440 * -1 to ignore file/directory (not used here)
442 int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level)
444 bool do_read = false;
445 int stat, data_stream;
447 DIGEST *digest = NULL;
448 DIGEST *signing_digest = NULL;
449 int digest_stream = STREAM_NONE;
450 SIGNATURE *sig = NULL;
451 bool has_file_data = false;
452 // TODO landonf: Allow the user to specify the digest algorithm
454 crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA256;
456 crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA1;
458 BSOCK *sd = jcr->store_bsock;
460 if (job_canceled(jcr)) {
464 jcr->num_files_examined++; /* bump total file count */
466 switch (ff_pkt->type) {
467 case FT_LNKSAVED: /* Hard linked, file already saved */
468 Dmsg2(130, "FT_LNKSAVED hard link: %s => %s\n", ff_pkt->fname, ff_pkt->link);
471 Dmsg1(130, "FT_REGE saving: %s\n", ff_pkt->fname);
472 if (no_dump(jcr, ff_pkt))
474 has_file_data = true;
477 Dmsg1(130, "FT_REG saving: %s\n", ff_pkt->fname);
478 if (no_dump(jcr, ff_pkt))
480 has_file_data = true;
483 Dmsg2(130, "FT_LNK saving: %s -> %s\n", ff_pkt->fname, ff_pkt->link);
486 jcr->num_files_examined--; /* correct file count */
487 if (no_dump(jcr, ff_pkt)) /* disable recursion on nodump directories */
488 ff_pkt->flags |= FO_NO_RECURSION;
489 return 1; /* not used */
491 Jmsg(jcr, M_INFO, 1, _(" Recursion turned off. Will not descend from %s into %s\n"),
492 ff_pkt->top_fname, ff_pkt->fname);
493 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
496 /* Suppress message for /dev filesystems */
497 if (!is_in_fileset(ff_pkt)) {
498 Jmsg(jcr, M_INFO, 1, _(" %s is a different filesystem. Will not descend from %s into %s\n"),
499 ff_pkt->fname, ff_pkt->top_fname, ff_pkt->fname);
501 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
504 Jmsg(jcr, M_INFO, 1, _(" Disallowed filesystem. Will not descend from %s into %s\n"),
505 ff_pkt->top_fname, ff_pkt->fname);
506 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
509 Jmsg(jcr, M_INFO, 1, _(" Disallowed drive type. Will not descend into %s\n"),
514 Dmsg1(130, "FT_DIREND: %s\n", ff_pkt->link);
517 Dmsg1(130, "FT_SPEC saving: %s\n", ff_pkt->fname);
518 if (S_ISSOCK(ff_pkt->statp.st_mode)) {
519 Jmsg(jcr, M_SKIPPED, 1, _(" Socket file skipped: %s\n"), ff_pkt->fname);
524 Dmsg1(130, "FT_RAW saving: %s\n", ff_pkt->fname);
525 has_file_data = true;
528 Dmsg1(130, "FT_FIFO saving: %s\n", ff_pkt->fname);
532 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not access %s: ERR=%s\n"), ff_pkt->fname,
533 be.bstrerror(ff_pkt->ff_errno));
539 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not follow link %s: ERR=%s\n"),
540 ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
546 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not stat %s: ERR=%s\n"), ff_pkt->fname,
547 be.bstrerror(ff_pkt->ff_errno));
553 Jmsg(jcr, M_SKIPPED, 1, _(" Unchanged file skipped: %s\n"), ff_pkt->fname);
556 Jmsg(jcr, M_NOTSAVED, 0, _(" Archive file not saved: %s\n"), ff_pkt->fname);
560 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not open directory %s: ERR=%s\n"),
561 ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
566 Jmsg(jcr, M_NOTSAVED, 0, _(" Unknown file type %d; not saved: %s\n"),
567 ff_pkt->type, ff_pkt->fname);
572 Dmsg1(130, "bfiled: sending %s to stored\n", ff_pkt->fname);
574 /* Digests and encryption are only useful if there's file data */
577 * Setup for digest handling. If this fails, the digest will be set to NULL
578 * and not used. Note, the digest (file hash) can be any one of the four
581 * The signing digest is a single algorithm depending on
582 * whether or not we have SHA2.
583 * ****FIXME**** the signing algoritm should really be
584 * determined a different way!!!!!! What happens if
585 * sha2 was available during backup but not restore?
587 if (ff_pkt->flags & FO_MD5) {
588 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_MD5);
589 digest_stream = STREAM_MD5_DIGEST;
591 } else if (ff_pkt->flags & FO_SHA1) {
592 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA1);
593 digest_stream = STREAM_SHA1_DIGEST;
595 } else if (ff_pkt->flags & FO_SHA256) {
596 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA256);
597 digest_stream = STREAM_SHA256_DIGEST;
599 } else if (ff_pkt->flags & FO_SHA512) {
600 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA512);
601 digest_stream = STREAM_SHA512_DIGEST;
604 /* Did digest initialization fail? */
605 if (digest_stream != STREAM_NONE && digest == NULL) {
606 Jmsg(jcr, M_WARNING, 0, _("%s digest initialization failed\n"),
607 stream_to_ascii(digest_stream));
611 * Set up signature digest handling. If this fails, the signature digest will be set to
614 // TODO landonf: We should really only calculate the digest once, for both verification and signing.
615 if (jcr->crypto.pki_sign) {
616 signing_digest = crypto_digest_new(jcr, signing_algorithm);
618 /* Full-stop if a failure occurred initializing the signature digest */
619 if (signing_digest == NULL) {
620 Jmsg(jcr, M_NOTSAVED, 0, _("%s signature digest initialization failed\n"),
621 stream_to_ascii(signing_algorithm));
627 /* Enable encryption */
628 if (jcr->crypto.pki_encrypt) {
629 ff_pkt->flags |= FO_ENCRYPT;
633 /* Initialize the file descriptor we use for data and other streams. */
635 if (ff_pkt->flags & FO_PORTABLE) {
636 set_portable_backup(&ff_pkt->bfd); /* disable Win32 BackupRead() */
638 if (ff_pkt->cmd_plugin) {
639 if (!set_cmd_plugin(&ff_pkt->bfd, jcr)) {
642 send_plugin_name(jcr, sd, true); /* signal start of plugin data */
645 /* Send attributes -- must be done after binit() */
646 if (!encode_and_send_attributes(jcr, ff_pkt, data_stream)) {
650 /* Set up the encryption context and send the session data to the SD */
651 if (has_file_data && jcr->crypto.pki_encrypt) {
652 if (!crypto_session_send(jcr, sd)) {
658 * Open any file with data that we intend to save, then save it.
660 * Note, if is_win32_backup, we must open the Directory so that
661 * the BackupRead will save its permissions and ownership streams.
663 if (ff_pkt->type != FT_LNKSAVED && S_ISREG(ff_pkt->statp.st_mode)) {
665 do_read = !is_portable_backup(&ff_pkt->bfd) || ff_pkt->statp.st_size > 0;
667 do_read = ff_pkt->statp.st_size > 0;
669 } else if (ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO ||
670 ff_pkt->type == FT_REPARSE ||
671 (!is_portable_backup(&ff_pkt->bfd) && ff_pkt->type == FT_DIREND)) {
674 if (ff_pkt->cmd_plugin) {
678 Dmsg1(100, "do_read=%d\n", do_read);
682 if (ff_pkt->type == FT_FIFO) {
683 tid = start_thread_timer(jcr, pthread_self(), 60);
687 int noatime = ff_pkt->flags & FO_NOATIME ? O_NOATIME : 0;
688 ff_pkt->bfd.reparse_point = ff_pkt->type == FT_REPARSE;
689 if (bopen(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY | noatime, 0) < 0) {
690 ff_pkt->ff_errno = errno;
692 Jmsg(jcr, M_NOTSAVED, 0, _(" Cannot open %s: ERR=%s.\n"), ff_pkt->fname,
696 stop_thread_timer(tid);
702 stop_thread_timer(tid);
706 stat = send_data(jcr, data_stream, ff_pkt, digest, signing_digest);
708 if (ff_pkt->flags & FO_CHKCHANGES) {
709 has_file_changed(jcr, ff_pkt);
712 bclose(&ff_pkt->bfd);
719 #ifdef HAVE_DARWIN_OS
720 /* Regular files can have resource forks and Finder Info */
721 if (ff_pkt->type != FT_LNKSAVED && (S_ISREG(ff_pkt->statp.st_mode) &&
722 ff_pkt->flags & FO_HFSPLUS)) {
723 if (ff_pkt->hfsinfo.rsrclength > 0) {
726 if (!bopen_rsrc(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0) < 0) {
727 ff_pkt->ff_errno = errno;
729 Jmsg(jcr, M_NOTSAVED, -1, _(" Cannot open resource fork for %s: ERR=%s.\n"),
730 ff_pkt->fname, be.bstrerror());
732 if (is_bopen(&ff_pkt->bfd)) {
733 bclose(&ff_pkt->bfd);
737 flags = ff_pkt->flags;
738 ff_pkt->flags &= ~(FO_GZIP|FO_SPARSE);
739 if (flags & FO_ENCRYPT) {
740 rsrc_stream = STREAM_ENCRYPTED_MACOS_FORK_DATA;
742 rsrc_stream = STREAM_MACOS_FORK_DATA;
744 stat = send_data(jcr, rsrc_stream, ff_pkt, digest, signing_digest);
745 ff_pkt->flags = flags;
746 bclose(&ff_pkt->bfd);
752 Dmsg1(300, "Saving Finder Info for \"%s\"\n", ff_pkt->fname);
753 sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_HFSPLUS_ATTRIBUTES);
754 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
755 memcpy(sd->msg, ff_pkt->hfsinfo.fndrinfo, 32);
758 crypto_digest_update(digest, (uint8_t *)sd->msg, sd->msglen);
760 if (signing_digest) {
761 crypto_digest_update(signing_digest, (uint8_t *)sd->msg, sd->msglen);
764 sd->signal(BNET_EOD);
768 if (ff_pkt->flags & FO_ACL) {
769 /* Read access ACLs for files, dirs and links */
770 if (!read_and_send_acl(jcr, BACL_TYPE_ACCESS, STREAM_UNIX_ACCESS_ACL)) {
773 /* Directories can have default ACLs too */
774 if (ff_pkt->type == FT_DIREND && (BACL_CAP & BACL_CAP_DEFAULTS_DIR)) {
775 if (!read_and_send_acl(jcr, BACL_TYPE_DEFAULT, STREAM_UNIX_DEFAULT_ACL)) {
781 /* Terminate the signing digest and send it to the Storage daemon */
782 if (signing_digest) {
785 if ((sig = crypto_sign_new(jcr)) == NULL) {
786 Jmsg(jcr, M_FATAL, 0, _("Failed to allocate memory for crypto signature.\n"));
790 if (!crypto_sign_add_signer(sig, signing_digest, jcr->crypto.pki_keypair)) {
791 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
795 /* Get signature size */
796 if (!crypto_sign_encode(sig, NULL, &size)) {
797 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
801 /* Grow the bsock buffer to fit our message if necessary */
802 if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
803 sd->msg = realloc_pool_memory(sd->msg, size);
806 /* Send our header */
807 sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_SIGNED_DIGEST);
808 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
810 /* Encode signature data */
811 if (!crypto_sign_encode(sig, (uint8_t *)sd->msg, &size)) {
812 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
818 sd->signal(BNET_EOD); /* end of checksum */
821 /* Terminate any digest and send it to Storage daemon */
825 sd->fsend("%ld %d 0", jcr->JobFiles, digest_stream);
826 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
828 size = CRYPTO_DIGEST_MAX_SIZE;
830 /* Grow the bsock buffer to fit our message if necessary */
831 if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
832 sd->msg = realloc_pool_memory(sd->msg, size);
835 if (!crypto_digest_finalize(digest, (uint8_t *)sd->msg, &size)) {
836 Jmsg(jcr, M_FATAL, 0, _("An error occurred finalizing signing the stream.\n"));
842 sd->signal(BNET_EOD); /* end of checksum */
844 if (ff_pkt->cmd_plugin) {
845 send_plugin_name(jcr, sd, false); /* signal end of plugin data */
849 rtnstat = 1; /* good return */
853 crypto_digest_free(digest);
855 if (signing_digest) {
856 crypto_digest_free(signing_digest);
859 crypto_sign_free(sig);
865 * Send data read from an already open file descriptor.
867 * We return 1 on sucess and 0 on errors.
870 * We use ff_pkt->statp.st_size when FO_SPARSE to know when to stop
872 * Currently this is not a problem as the only other stream, resource forks,
873 * are not handled as sparse files.
875 int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest,
876 DIGEST *signing_digest)
878 BSOCK *sd = jcr->store_bsock;
879 uint64_t fileAddr = 0; /* file address */
881 int32_t rsize = jcr->buf_size; /* read buffer size */
883 CIPHER_CONTEXT *cipher_ctx = NULL; /* Quell bogus uninitialized warnings */
884 const uint8_t *cipher_input;
885 uint32_t cipher_input_len;
886 uint32_t cipher_block_size;
887 uint32_t encrypted_len;
888 #ifdef FD_NO_SEND_TEST
893 rbuf = sd->msg; /* read buffer */
894 wbuf = sd->msg; /* write buffer */
895 cipher_input = (uint8_t *)rbuf; /* encrypt uncompressed data */
897 Dmsg1(300, "Saving data, type=%d\n", ff_pkt->type);
900 uLong compress_len = 0;
901 uLong max_compress_len = 0;
902 const Bytef *cbuf = NULL;
905 if (ff_pkt->flags & FO_GZIP) {
906 if (ff_pkt->flags & FO_SPARSE) {
907 cbuf = (Bytef *)jcr->compress_buf + SPARSE_FADDR_SIZE;
908 max_compress_len = jcr->compress_buf_size - SPARSE_FADDR_SIZE;
910 cbuf = (Bytef *)jcr->compress_buf;
911 max_compress_len = jcr->compress_buf_size; /* set max length */
913 wbuf = jcr->compress_buf; /* compressed output here */
914 cipher_input = (uint8_t *)jcr->compress_buf; /* encrypt compressed data */
917 * Only change zlib parameters if there is no pending operation.
918 * This should never happen as deflatereset is called after each
922 if (((z_stream*)jcr->pZLIB_compress_workset)->total_in == 0) {
923 /* set gzip compression level - must be done per file */
924 if ((zstat=deflateParams((z_stream*)jcr->pZLIB_compress_workset,
925 ff_pkt->GZIP_level, Z_DEFAULT_STRATEGY)) != Z_OK) {
926 Jmsg(jcr, M_FATAL, 0, _("Compression deflateParams error: %d\n"), zstat);
927 set_jcr_job_status(jcr, JS_ErrorTerminated);
933 const uint32_t max_compress_len = 0;
936 if (ff_pkt->flags & FO_ENCRYPT) {
937 if (ff_pkt->flags & FO_SPARSE) {
938 Jmsg0(jcr, M_FATAL, 0, _("Encrypting sparse data not supported.\n"));
941 /* Allocate the cipher context */
942 if ((cipher_ctx = crypto_cipher_new(jcr->crypto.pki_session, true,
943 &cipher_block_size)) == NULL) {
944 /* Shouldn't happen! */
945 Jmsg0(jcr, M_FATAL, 0, _("Failed to initialize encryption context.\n"));
950 * Grow the crypto buffer, if necessary.
951 * crypto_cipher_update() will buffer up to (cipher_block_size - 1).
952 * We grow crypto_buf to the maximum number of blocks that
953 * could be returned for the given read buffer size.
954 * (Using the larger of either rsize or max_compress_len)
956 jcr->crypto.crypto_buf = check_pool_memory_size(jcr->crypto.crypto_buf,
957 (MAX(rsize + (int)sizeof(uint32_t), (int32_t)max_compress_len) +
958 cipher_block_size - 1) / cipher_block_size * cipher_block_size);
960 wbuf = jcr->crypto.crypto_buf; /* Encrypted, possibly compressed output here. */
964 * Send Data header to Storage daemon
965 * <file-index> <stream> <info>
967 if (!sd->fsend("%ld %d 0", jcr->JobFiles, stream)) {
968 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
972 Dmsg1(300, ">stored: datahdr %s\n", sd->msg);
975 * Make space at beginning of buffer for fileAddr because this
976 * same buffer will be used for writing if compression is off.
978 if (ff_pkt->flags & FO_SPARSE) {
979 rbuf += SPARSE_FADDR_SIZE;
980 rsize -= SPARSE_FADDR_SIZE;
981 #ifdef HAVE_FREEBSD_OS
983 * To read FreeBSD partitions, the read size must be
986 rsize = (rsize/512) * 512;
990 /* a RAW device read on win32 only works if the buffer is a multiple of 512 */
992 if (S_ISBLK(ff_pkt->statp.st_mode))
993 rsize = (rsize/512) * 512;
999 while ((sd->msglen=(uint32_t)bread(&ff_pkt->bfd, rbuf, rsize)) > 0) {
1001 /* Check for sparse blocks */
1002 if (ff_pkt->flags & FO_SPARSE) {
1004 bool haveBlock = true;
1005 if (sd->msglen == rsize &&
1006 fileAddr+sd->msglen < (uint64_t)ff_pkt->statp.st_size ||
1007 ((ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO) &&
1008 (uint64_t)ff_pkt->statp.st_size == 0)) {
1009 haveBlock = !is_buf_zero(rbuf, rsize);
1012 ser_begin(wbuf, SPARSE_FADDR_SIZE);
1013 ser_uint64(fileAddr); /* store fileAddr in begin of buffer */
1015 fileAddr += sd->msglen; /* update file address */
1017 continue; /* skip block of zeros */
1021 jcr->ReadBytes += sd->msglen; /* count bytes read */
1023 /* Uncompressed cipher input length */
1024 cipher_input_len = sd->msglen;
1026 /* Update checksum if requested */
1028 crypto_digest_update(digest, (uint8_t *)rbuf, sd->msglen);
1031 /* Update signing digest if requested */
1032 if (signing_digest) {
1033 crypto_digest_update(signing_digest, (uint8_t *)rbuf, sd->msglen);
1037 /* Do compression if turned on */
1038 if (ff_pkt->flags & FO_GZIP && jcr->pZLIB_compress_workset) {
1039 Dmsg3(400, "cbuf=0x%x rbuf=0x%x len=%u\n", cbuf, rbuf, sd->msglen);
1041 ((z_stream*)jcr->pZLIB_compress_workset)->next_in = (Bytef *)rbuf;
1042 ((z_stream*)jcr->pZLIB_compress_workset)->avail_in = sd->msglen;
1043 ((z_stream*)jcr->pZLIB_compress_workset)->next_out = (Bytef *)cbuf;
1044 ((z_stream*)jcr->pZLIB_compress_workset)->avail_out = max_compress_len;
1046 if ((zstat=deflate((z_stream*)jcr->pZLIB_compress_workset, Z_FINISH)) != Z_STREAM_END) {
1047 Jmsg(jcr, M_FATAL, 0, _("Compression deflate error: %d\n"), zstat);
1048 set_jcr_job_status(jcr, JS_ErrorTerminated);
1051 compress_len = ((z_stream*)jcr->pZLIB_compress_workset)->total_out;
1052 /* reset zlib stream to be able to begin from scratch again */
1053 if ((zstat=deflateReset((z_stream*)jcr->pZLIB_compress_workset)) != Z_OK) {
1054 Jmsg(jcr, M_FATAL, 0, _("Compression deflateReset error: %d\n"), zstat);
1055 set_jcr_job_status(jcr, JS_ErrorTerminated);
1059 Dmsg2(400, "compressed len=%d uncompressed len=%d\n", compress_len,
1062 sd->msglen = compress_len; /* set compressed length */
1063 cipher_input_len = compress_len;
1067 * Note, here we prepend the current record length to the beginning
1068 * of the encrypted data. This is because both sparse and compression
1069 * restore handling want records returned to them with exactly the
1070 * same number of bytes that were processed in the backup handling.
1071 * That is, both are block filters rather than a stream. When doing
1072 * compression, the compression routines may buffer data, so that for
1073 * any one record compressed, when it is decompressed the same size
1074 * will not be obtained. Of course, the buffered data eventually comes
1075 * out in subsequent crypto_cipher_update() calls or at least
1076 * when crypto_cipher_finalize() is called. Unfortunately, this
1077 * "feature" of encryption enormously complicates the restore code.
1079 if (ff_pkt->flags & FO_ENCRYPT) {
1080 uint32_t initial_len = 0;
1083 if (ff_pkt->flags & FO_SPARSE) {
1084 cipher_input_len += SPARSE_FADDR_SIZE;
1087 /* Encrypt the length of the input block */
1088 uint8_t packet_len[sizeof(uint32_t)];
1090 ser_begin(packet_len, sizeof(uint32_t));
1091 ser_uint32(cipher_input_len); /* store data len in begin of buffer */
1092 Dmsg1(20, "Encrypt len=%d\n", cipher_input_len);
1094 if (!crypto_cipher_update(cipher_ctx, packet_len, sizeof(packet_len),
1095 (uint8_t *)jcr->crypto.crypto_buf, &initial_len)) {
1096 /* Encryption failed. Shouldn't happen. */
1097 Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
1101 /* Encrypt the input block */
1102 if (crypto_cipher_update(cipher_ctx, cipher_input, cipher_input_len,
1103 (uint8_t *)&jcr->crypto.crypto_buf[initial_len], &encrypted_len)) {
1104 if ((initial_len + encrypted_len) == 0) {
1105 /* No full block of data available, read more data */
1108 Dmsg2(400, "encrypted len=%d unencrypted len=%d\n", encrypted_len,
1110 sd->msglen = initial_len + encrypted_len; /* set encrypted length */
1112 /* Encryption failed. Shouldn't happen. */
1113 Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
1118 /* Send the buffer to the Storage daemon */
1119 if (ff_pkt->flags & FO_SPARSE) {
1120 sd->msglen += SPARSE_FADDR_SIZE; /* include fileAddr in size */
1122 sd->msg = wbuf; /* set correct write buffer */
1124 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1128 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
1130 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
1131 sd->msg = msgsave; /* restore read buffer */
1133 } /* end while read file data */
1135 if (sd->msglen < 0) { /* error */
1137 Jmsg(jcr, M_ERROR, 0, _("Read error on file %s. ERR=%s\n"),
1138 ff_pkt->fname, be.bstrerror(ff_pkt->bfd.berrno));
1139 if (jcr->Errors++ > 1000) { /* insanity check */
1140 Jmsg(jcr, M_FATAL, 0, _("Too many errors.\n"));
1142 } else if (ff_pkt->flags & FO_ENCRYPT) {
1144 * For encryption, we must call finalize to push out any
1147 if (!crypto_cipher_finalize(cipher_ctx, (uint8_t *)jcr->crypto.crypto_buf,
1149 /* Padding failed. Shouldn't happen. */
1150 Jmsg(jcr, M_FATAL, 0, _("Encryption padding error\n"));
1154 /* Note, on SSL pre-0.9.7, there is always some output */
1155 if (encrypted_len > 0) {
1156 sd->msglen = encrypted_len; /* set encrypted length */
1157 sd->msg = jcr->crypto.crypto_buf; /* set correct write buffer */
1159 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1163 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
1164 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
1165 sd->msg = msgsave; /* restore bnet buffer */
1169 if (!sd->signal(BNET_EOD)) { /* indicate end of file data */
1170 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1175 /* Free the cipher context */
1177 crypto_cipher_free(cipher_ctx);
1182 /* Free the cipher context */
1184 crypto_cipher_free(cipher_ctx);
1187 sd->msg = msgsave; /* restore bnet buffer */
1193 * Read and send an ACL for the last encountered file.
1195 static bool read_and_send_acl(JCR *jcr, int acltype, int stream)
1198 BSOCK *sd = jcr->store_bsock;
1201 #ifdef FD_NO_SEND_TEST
1205 len = bacl_get(jcr, acltype);
1207 Jmsg1(jcr, M_WARNING, 0, _("Error reading ACL of %s\n"), jcr->last_fname);
1211 return true; /* no ACL */
1215 if (!sd->fsend("%ld %d 0", jcr->JobFiles, stream)) {
1216 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1221 /* Send the buffer to the storage deamon */
1222 Dmsg2(400, "Backing up ACL type 0x%2x <%s>\n", acltype, jcr->acl_text);
1224 sd->msg = jcr->acl_text;
1225 sd->msglen = len + 1;
1229 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1234 jcr->JobBytes += sd->msglen;
1236 if (!sd->signal(BNET_EOD)) {
1237 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1242 Dmsg1(200, "ACL of file: %s successfully backed up!\n", jcr->last_fname);
1247 static bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream)
1249 BSOCK *sd = jcr->store_bsock;
1250 char attribs[MAXSTRING];
1251 char attribsEx[MAXSTRING];
1254 #ifdef FD_NO_SEND_TEST
1258 Dmsg1(300, "encode_and_send_attrs fname=%s\n", ff_pkt->fname);
1259 /* Find what data stream we will use, then encode the attributes */
1260 if ((data_stream = select_data_stream(ff_pkt)) == STREAM_NONE) {
1261 /* This should not happen */
1262 Jmsg0(jcr, M_FATAL, 0, _("Invalid file flags, no supported data stream type.\n"));
1265 encode_stat(attribs, ff_pkt, data_stream);
1267 /* Now possibly extend the attributes */
1268 attr_stream = encode_attribsEx(jcr, attribsEx, ff_pkt);
1270 Dmsg3(300, "File %s\nattribs=%s\nattribsEx=%s\n", ff_pkt->fname, attribs, attribsEx);
1273 jcr->JobFiles++; /* increment number of files sent */
1274 ff_pkt->FileIndex = jcr->JobFiles; /* return FileIndex */
1275 pm_strcpy(jcr->last_fname, ff_pkt->fname);
1279 * Send Attributes header to Storage daemon
1280 * <file-index> <stream> <info>
1282 if (!sd->fsend("%ld %d 0", jcr->JobFiles, attr_stream)) {
1283 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1287 Dmsg1(300, ">stored: attrhdr %s\n", sd->msg);
1290 * Send file attributes to Storage daemon
1293 * Filename (full path)
1294 * Encoded attributes
1295 * Link name (if type==FT_LNK or FT_LNKSAVED)
1296 * Encoded extended-attributes (for Win32)
1298 * For a directory, link is the same as fname, but with trailing
1299 * slash. For a linked file, link is the link.
1301 if (ff_pkt->type != FT_DELETED) { /* already stripped */
1304 if (ff_pkt->type == FT_LNK || ff_pkt->type == FT_LNKSAVED) {
1305 Dmsg2(300, "Link %s to %s\n", ff_pkt->fname, ff_pkt->link);
1306 stat = sd->fsend("%ld %d %s%c%s%c%s%c%s%c", jcr->JobFiles,
1307 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, ff_pkt->link, 0,
1309 } else if (ff_pkt->type == FT_DIREND || ff_pkt->type == FT_REPARSE) {
1310 /* Here link is the canonical filename (i.e. with trailing slash) */
1311 stat = sd->fsend("%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
1312 ff_pkt->type, ff_pkt->link, 0, attribs, 0, 0, attribsEx, 0);
1314 stat = sd->fsend("%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
1315 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, 0, attribsEx, 0);
1317 if (ff_pkt->type != FT_DELETED) {
1318 unstrip_path(ff_pkt);
1321 Dmsg2(300, ">stored: attr len=%d: %s\n", sd->msglen, sd->msg);
1323 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1327 sd->signal(BNET_EOD); /* indicate end of attributes data */
1332 * Do in place strip of path
1334 static bool do_strip(int count, char *in)
1340 /* Copy to first path separator -- Win32 might have c: ... */
1341 while (*in && !IsPathSeparator(*in)) {
1345 numsep++; /* one separator seen */
1346 for (stripped=0; stripped<count && *in; stripped++) {
1347 while (*in && !IsPathSeparator(*in)) {
1348 in++; /* skip chars */
1351 numsep++; /* count separators seen */
1352 in++; /* skip separator */
1356 while (*in) { /* copy to end */
1357 if (IsPathSeparator(*in)) {
1363 Dmsg4(500, "stripped=%d count=%d numsep=%d sep>count=%d\n",
1364 stripped, count, numsep, numsep>count);
1365 return stripped==count && numsep>count;
1369 * If requested strip leading components of the path so that we can
1370 * save file as if it came from a subdirectory. This is most useful
1371 * for dealing with snapshots, by removing the snapshot directory, or
1372 * in handling vendor migrations where files have been restored with
1373 * a vendor product into a subdirectory.
1375 static void strip_path(FF_PKT *ff_pkt)
1377 if (!(ff_pkt->flags & FO_STRIPPATH) || ff_pkt->strip_path <= 0) {
1378 Dmsg1(200, "No strip for %s\n", ff_pkt->fname);
1381 if (!ff_pkt->fname_save) {
1382 ff_pkt->fname_save = get_pool_memory(PM_FNAME);
1383 ff_pkt->link_save = get_pool_memory(PM_FNAME);
1385 pm_strcpy(ff_pkt->fname_save, ff_pkt->fname);
1388 * Strip path. If it doesn't succeed put it back. If
1389 * it does, and there is a different link string,
1390 * attempt to strip the link. If it fails, back them
1392 * Do not strip symlinks.
1393 * I.e. if either stripping fails don't strip anything.
1395 if (do_strip(ff_pkt->strip_path, ff_pkt->fname)) {
1396 /* Strip links but not symlinks */
1397 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1398 pm_strcpy(ff_pkt->link_save, ff_pkt->link);
1399 if (!do_strip(ff_pkt->strip_path, ff_pkt->link)) {
1400 strcpy(ff_pkt->link, ff_pkt->link_save);
1401 strcpy(ff_pkt->fname, ff_pkt->fname_save);
1405 strcpy(ff_pkt->fname, ff_pkt->fname_save);
1407 Dmsg2(200, "fname=%s stripped=%s\n", ff_pkt->fname_save, ff_pkt->fname);
1410 static void unstrip_path(FF_PKT *ff_pkt)
1412 if (!(ff_pkt->flags & FO_STRIPPATH) || ff_pkt->strip_path <= 0) {
1415 strcpy(ff_pkt->fname, ff_pkt->fname_save);
1416 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1417 strcpy(ff_pkt->link, ff_pkt->link_save);