2 Bacula® - The Network Backup Solution
4 Copyright (C) 2000-2009 Free Software Foundation Europe e.V.
6 The main author of Bacula is Kern Sibbald, with contributions from
7 many others, a complete list can be found in the file AUTHORS.
8 This program is Free Software; you can redistribute it and/or
9 modify it under the terms of version two of the GNU General Public
10 License as published by the Free Software Foundation and included
13 This program is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
23 Bacula® is a registered trademark of Kern Sibbald.
24 The licensor of Bacula is the Free Software Foundation Europe
25 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
26 Switzerland, email:ftf@fsfeurope.org.
29 * Bacula File Daemon backup.c send file attributes and data
30 * to the Storage daemon.
32 * Kern Sibbald, March MM
41 /* Forward referenced functions */
42 int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level);
43 static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, DIGEST *signature_digest);
44 bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream);
45 static bool crypto_session_start(JCR *jcr);
46 static void crypto_session_end(JCR *jcr);
47 static bool crypto_session_send(JCR *jcr, BSOCK *sd);
50 * Find all the requested files and send them
51 * to the Storage daemon.
53 * Note, we normally carry on a one-way
54 * conversation from this point on with the SD, simply blasting
55 * data to him. To properly know what is going on, we
56 * also run a "heartbeat" monitor which reads the socket and
57 * reacts accordingly (at the moment it has nothing to do
58 * except echo the heartbeat to the Director).
61 bool blast_data_to_storage_daemon(JCR *jcr, char *addr)
65 // TODO landonf: Allow user to specify encryption algorithm
67 sd = jcr->store_bsock;
69 set_jcr_job_status(jcr, JS_Running);
71 Dmsg1(300, "bfiled: opened data connection %d to stored\n", sd->m_fd);
74 CLIENT *client = (CLIENT *)GetNextRes(R_CLIENT, NULL);
78 buf_size = client->max_network_buffer_size;
80 buf_size = 0; /* use default */
82 if (!sd->set_buffer_size(buf_size, BNET_SETBUF_WRITE)) {
83 set_jcr_job_status(jcr, JS_ErrorTerminated);
84 Jmsg(jcr, M_FATAL, 0, _("Cannot set buffer size FD->SD.\n"));
88 jcr->buf_size = sd->msglen;
89 /* Adjust for compression so that output buffer is
90 * 12 bytes + 0.1% larger than input buffer plus 18 bytes.
91 * This gives a bit extra plus room for the sparse addr if any.
92 * Note, we adjust the read size to be smaller so that the
93 * same output buffer can be used without growing it.
95 * The zlib compression workset is initialized here to minimize
96 * the "per file" load. The jcr member is only set, if the init
99 jcr->compress_buf_size = jcr->buf_size + ((jcr->buf_size+999) / 1000) + 30;
100 jcr->compress_buf = get_memory(jcr->compress_buf_size);
103 z_stream *pZlibStream = (z_stream*)malloc(sizeof(z_stream));
105 pZlibStream->zalloc = Z_NULL;
106 pZlibStream->zfree = Z_NULL;
107 pZlibStream->opaque = Z_NULL;
108 pZlibStream->state = Z_NULL;
110 if (deflateInit(pZlibStream, Z_DEFAULT_COMPRESSION) == Z_OK) {
111 jcr->pZLIB_compress_workset = pZlibStream;
118 if (!crypto_session_start(jcr)) {
122 set_find_options((FF_PKT *)jcr->ff, jcr->incremental, jcr->mtime);
124 /* in accurate mode, we overwrite the find_one check function */
126 set_find_changed_function((FF_PKT *)jcr->ff, accurate_check_file);
129 start_heartbeat_monitor(jcr);
131 jcr->acl_data = get_pool_memory(PM_MESSAGE);
132 jcr->xattr_data = get_pool_memory(PM_MESSAGE);
134 /* Subroutine save_file() is called for each file */
135 if (!find_files(jcr, (FF_PKT *)jcr->ff, save_file, plugin_save)) {
136 ok = false; /* error */
137 set_jcr_job_status(jcr, JS_ErrorTerminated);
140 accurate_send_deleted_list(jcr); /* send deleted list to SD */
142 stop_heartbeat_monitor(jcr);
144 sd->signal(BNET_EOD); /* end of sending data */
147 free_pool_memory(jcr->acl_data);
148 jcr->acl_data = NULL;
150 if (jcr->xattr_data) {
151 free_pool_memory(jcr->xattr_data);
152 jcr->xattr_data = NULL;
158 if (jcr->compress_buf) {
159 free_pool_memory(jcr->compress_buf);
160 jcr->compress_buf = NULL;
162 if (jcr->pZLIB_compress_workset) {
163 /* Free the zlib stream */
165 deflateEnd((z_stream *)jcr->pZLIB_compress_workset);
167 free (jcr->pZLIB_compress_workset);
168 jcr->pZLIB_compress_workset = NULL;
170 crypto_session_end(jcr);
173 Dmsg1(100, "end blast_data ok=%d\n", ok);
177 static bool crypto_session_start(JCR *jcr)
179 crypto_cipher_t cipher = CRYPTO_CIPHER_AES_128_CBC;
182 * Create encryption session data and a cached, DER-encoded session data
183 * structure. We use a single session key for each backup, so we'll encode
184 * the session data only once.
186 if (jcr->crypto.pki_encrypt) {
189 /* Create per-job session encryption context */
190 jcr->crypto.pki_session = crypto_session_new(cipher, jcr->crypto.pki_recipients);
192 /* Get the session data size */
193 if (!crypto_session_encode(jcr->crypto.pki_session, (uint8_t *)0, &size)) {
194 Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n"));
198 /* Allocate buffer */
199 jcr->crypto.pki_session_encoded = get_memory(size);
201 /* Encode session data */
202 if (!crypto_session_encode(jcr->crypto.pki_session, (uint8_t *)jcr->crypto.pki_session_encoded, &size)) {
203 Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n"));
207 /* ... and store the encoded size */
208 jcr->crypto.pki_session_encoded_size = size;
210 /* Allocate the encryption/decryption buffer */
211 jcr->crypto.crypto_buf = get_memory(CRYPTO_CIPHER_MAX_BLOCK_SIZE);
216 static void crypto_session_end(JCR *jcr)
218 if (jcr->crypto.crypto_buf) {
219 free_pool_memory(jcr->crypto.crypto_buf);
220 jcr->crypto.crypto_buf = NULL;
222 if (jcr->crypto.pki_session) {
223 crypto_session_free(jcr->crypto.pki_session);
225 if (jcr->crypto.pki_session_encoded) {
226 free_pool_memory(jcr->crypto.pki_session_encoded);
227 jcr->crypto.pki_session_encoded = NULL;
231 static bool crypto_session_send(JCR *jcr, BSOCK *sd)
235 /* Send our header */
236 Dmsg2(100, "Send hdr fi=%ld stream=%d\n", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA);
237 sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA);
240 sd->msg = jcr->crypto.pki_session_encoded;
241 sd->msglen = jcr->crypto.pki_session_encoded_size;
242 jcr->JobBytes += sd->msglen;
244 Dmsg1(100, "Send data len=%d\n", sd->msglen);
247 sd->signal(BNET_EOD);
253 * Called here by find() for each file included.
254 * This is a callback. The original is find_files() above.
256 * Send the file and its data to the Storage daemon.
260 * -1 to ignore file/directory (not used here)
262 int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level)
264 bool do_read = false;
265 int stat, data_stream;
267 DIGEST *digest = NULL;
268 DIGEST *signing_digest = NULL;
269 int digest_stream = STREAM_NONE;
270 SIGNATURE *sig = NULL;
271 bool has_file_data = false;
272 // TODO landonf: Allow the user to specify the digest algorithm
274 crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA256;
276 crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA1;
278 BSOCK *sd = jcr->store_bsock;
280 if (job_canceled(jcr)) {
284 jcr->num_files_examined++; /* bump total file count */
286 switch (ff_pkt->type) {
287 case FT_LNKSAVED: /* Hard linked, file already saved */
288 Dmsg2(130, "FT_LNKSAVED hard link: %s => %s\n", ff_pkt->fname, ff_pkt->link);
291 Dmsg1(130, "FT_REGE saving: %s\n", ff_pkt->fname);
292 has_file_data = true;
295 Dmsg1(130, "FT_REG saving: %s\n", ff_pkt->fname);
296 has_file_data = true;
299 Dmsg2(130, "FT_LNK saving: %s -> %s\n", ff_pkt->fname, ff_pkt->link);
302 jcr->num_files_examined--; /* correct file count */
303 return 1; /* not used */
305 Jmsg(jcr, M_INFO, 1, _(" Recursion turned off. Will not descend from %s into %s\n"),
306 ff_pkt->top_fname, ff_pkt->fname);
307 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
310 /* Suppress message for /dev filesystems */
311 if (!is_in_fileset(ff_pkt)) {
312 Jmsg(jcr, M_INFO, 1, _(" %s is a different filesystem. Will not descend from %s into %s\n"),
313 ff_pkt->fname, ff_pkt->top_fname, ff_pkt->fname);
315 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
318 Jmsg(jcr, M_INFO, 1, _(" Disallowed filesystem. Will not descend from %s into %s\n"),
319 ff_pkt->top_fname, ff_pkt->fname);
320 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
323 Jmsg(jcr, M_INFO, 1, _(" Disallowed drive type. Will not descend into %s\n"),
328 Dmsg1(130, "FT_DIREND: %s\n", ff_pkt->link);
331 Dmsg1(130, "FT_SPEC saving: %s\n", ff_pkt->fname);
332 if (S_ISSOCK(ff_pkt->statp.st_mode)) {
333 Jmsg(jcr, M_SKIPPED, 1, _(" Socket file skipped: %s\n"), ff_pkt->fname);
338 Dmsg1(130, "FT_RAW saving: %s\n", ff_pkt->fname);
339 has_file_data = true;
342 Dmsg1(130, "FT_FIFO saving: %s\n", ff_pkt->fname);
346 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not access \"%s\": ERR=%s\n"), ff_pkt->fname,
347 be.bstrerror(ff_pkt->ff_errno));
353 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not follow link \"%s\": ERR=%s\n"),
354 ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
360 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not stat \"%s\": ERR=%s\n"), ff_pkt->fname,
361 be.bstrerror(ff_pkt->ff_errno));
367 Jmsg(jcr, M_SKIPPED, 1, _(" Unchanged file skipped: %s\n"), ff_pkt->fname);
370 Jmsg(jcr, M_NOTSAVED, 0, _(" Archive file not saved: %s\n"), ff_pkt->fname);
374 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not open directory \"%s\": ERR=%s\n"),
375 ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
380 Jmsg(jcr, M_NOTSAVED, 0, _(" Unknown file type %d; not saved: %s\n"),
381 ff_pkt->type, ff_pkt->fname);
386 Dmsg1(130, "bfiled: sending %s to stored\n", ff_pkt->fname);
388 /* Digests and encryption are only useful if there's file data */
391 * Setup for digest handling. If this fails, the digest will be set to NULL
392 * and not used. Note, the digest (file hash) can be any one of the four
395 * The signing digest is a single algorithm depending on
396 * whether or not we have SHA2.
397 * ****FIXME**** the signing algoritm should really be
398 * determined a different way!!!!!! What happens if
399 * sha2 was available during backup but not restore?
401 if (ff_pkt->flags & FO_MD5) {
402 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_MD5);
403 digest_stream = STREAM_MD5_DIGEST;
405 } else if (ff_pkt->flags & FO_SHA1) {
406 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA1);
407 digest_stream = STREAM_SHA1_DIGEST;
409 } else if (ff_pkt->flags & FO_SHA256) {
410 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA256);
411 digest_stream = STREAM_SHA256_DIGEST;
413 } else if (ff_pkt->flags & FO_SHA512) {
414 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA512);
415 digest_stream = STREAM_SHA512_DIGEST;
418 /* Did digest initialization fail? */
419 if (digest_stream != STREAM_NONE && digest == NULL) {
420 Jmsg(jcr, M_WARNING, 0, _("%s digest initialization failed\n"),
421 stream_to_ascii(digest_stream));
425 * Set up signature digest handling. If this fails, the signature digest will be set to
428 // TODO landonf: We should really only calculate the digest once, for both verification and signing.
429 if (jcr->crypto.pki_sign) {
430 signing_digest = crypto_digest_new(jcr, signing_algorithm);
432 /* Full-stop if a failure occurred initializing the signature digest */
433 if (signing_digest == NULL) {
434 Jmsg(jcr, M_NOTSAVED, 0, _("%s signature digest initialization failed\n"),
435 stream_to_ascii(signing_algorithm));
441 /* Enable encryption */
442 if (jcr->crypto.pki_encrypt) {
443 ff_pkt->flags |= FO_ENCRYPT;
447 /* Initialize the file descriptor we use for data and other streams. */
449 if (ff_pkt->flags & FO_PORTABLE) {
450 set_portable_backup(&ff_pkt->bfd); /* disable Win32 BackupRead() */
452 if (ff_pkt->cmd_plugin) {
453 if (!set_cmd_plugin(&ff_pkt->bfd, jcr)) {
456 send_plugin_name(jcr, sd, true); /* signal start of plugin data */
459 /* Send attributes -- must be done after binit() */
460 if (!encode_and_send_attributes(jcr, ff_pkt, data_stream)) {
464 /* Set up the encryption context and send the session data to the SD */
465 if (has_file_data && jcr->crypto.pki_encrypt) {
466 if (!crypto_session_send(jcr, sd)) {
472 * Open any file with data that we intend to save, then save it.
474 * Note, if is_win32_backup, we must open the Directory so that
475 * the BackupRead will save its permissions and ownership streams.
477 if (ff_pkt->type != FT_LNKSAVED && S_ISREG(ff_pkt->statp.st_mode)) {
479 do_read = !is_portable_backup(&ff_pkt->bfd) || ff_pkt->statp.st_size > 0;
481 do_read = ff_pkt->statp.st_size > 0;
483 } else if (ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO ||
484 ff_pkt->type == FT_REPARSE ||
485 (!is_portable_backup(&ff_pkt->bfd) && ff_pkt->type == FT_DIREND)) {
488 if (ff_pkt->cmd_plugin) {
492 Dmsg1(400, "do_read=%d\n", do_read);
496 if (ff_pkt->type == FT_FIFO) {
497 tid = start_thread_timer(jcr, pthread_self(), 60);
501 int noatime = ff_pkt->flags & FO_NOATIME ? O_NOATIME : 0;
502 ff_pkt->bfd.reparse_point = ff_pkt->type == FT_REPARSE;
503 if (bopen(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY | noatime, 0) < 0) {
504 ff_pkt->ff_errno = errno;
506 Jmsg(jcr, M_NOTSAVED, 0, _(" Cannot open \"%s\": ERR=%s.\n"), ff_pkt->fname,
510 stop_thread_timer(tid);
516 stop_thread_timer(tid);
520 stat = send_data(jcr, data_stream, ff_pkt, digest, signing_digest);
522 if (ff_pkt->flags & FO_CHKCHANGES) {
523 has_file_changed(jcr, ff_pkt);
526 bclose(&ff_pkt->bfd);
533 #ifdef HAVE_DARWIN_OS
534 /* Regular files can have resource forks and Finder Info */
535 if (ff_pkt->type != FT_LNKSAVED && (S_ISREG(ff_pkt->statp.st_mode) &&
536 ff_pkt->flags & FO_HFSPLUS)) {
537 if (ff_pkt->hfsinfo.rsrclength > 0) {
540 if (!bopen_rsrc(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0) < 0) {
541 ff_pkt->ff_errno = errno;
543 Jmsg(jcr, M_NOTSAVED, -1, _(" Cannot open resource fork for \"%s\": ERR=%s.\n"),
544 ff_pkt->fname, be.bstrerror());
546 if (is_bopen(&ff_pkt->bfd)) {
547 bclose(&ff_pkt->bfd);
551 flags = ff_pkt->flags;
552 ff_pkt->flags &= ~(FO_GZIP|FO_SPARSE);
553 if (flags & FO_ENCRYPT) {
554 rsrc_stream = STREAM_ENCRYPTED_MACOS_FORK_DATA;
556 rsrc_stream = STREAM_MACOS_FORK_DATA;
558 stat = send_data(jcr, rsrc_stream, ff_pkt, digest, signing_digest);
559 ff_pkt->flags = flags;
560 bclose(&ff_pkt->bfd);
566 Dmsg1(300, "Saving Finder Info for \"%s\"\n", ff_pkt->fname);
567 sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_HFSPLUS_ATTRIBUTES);
568 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
569 pm_memcpy(sd->msg, ff_pkt->hfsinfo.fndrinfo, 32);
572 crypto_digest_update(digest, (uint8_t *)sd->msg, sd->msglen);
574 if (signing_digest) {
575 crypto_digest_update(signing_digest, (uint8_t *)sd->msg, sd->msglen);
578 sd->signal(BNET_EOD);
583 * Save ACLs for anything not being a symlink and not being a plugin.
585 if (!ff_pkt->cmd_plugin) {
586 if (ff_pkt->flags & FO_ACL && ff_pkt->type != FT_LNK) {
587 if (!build_acl_streams(jcr, ff_pkt))
593 * Save Extended Attributes for all files not being a plugin.
595 if (!ff_pkt->cmd_plugin) {
596 if (ff_pkt->flags & FO_XATTR) {
597 if (!build_xattr_streams(jcr, ff_pkt))
602 /* Terminate the signing digest and send it to the Storage daemon */
603 if (signing_digest) {
606 if ((sig = crypto_sign_new(jcr)) == NULL) {
607 Jmsg(jcr, M_FATAL, 0, _("Failed to allocate memory for crypto signature.\n"));
611 if (!crypto_sign_add_signer(sig, signing_digest, jcr->crypto.pki_keypair)) {
612 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
616 /* Get signature size */
617 if (!crypto_sign_encode(sig, NULL, &size)) {
618 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
622 /* Grow the bsock buffer to fit our message if necessary */
623 if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
624 sd->msg = realloc_pool_memory(sd->msg, size);
627 /* Send our header */
628 sd->fsend("%ld %ld 0", jcr->JobFiles, STREAM_SIGNED_DIGEST);
629 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
631 /* Encode signature data */
632 if (!crypto_sign_encode(sig, (uint8_t *)sd->msg, &size)) {
633 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
639 sd->signal(BNET_EOD); /* end of checksum */
642 /* Terminate any digest and send it to Storage daemon */
646 sd->fsend("%ld %d 0", jcr->JobFiles, digest_stream);
647 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
649 size = CRYPTO_DIGEST_MAX_SIZE;
651 /* Grow the bsock buffer to fit our message if necessary */
652 if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
653 sd->msg = realloc_pool_memory(sd->msg, size);
656 if (!crypto_digest_finalize(digest, (uint8_t *)sd->msg, &size)) {
657 Jmsg(jcr, M_FATAL, 0, _("An error occurred finalizing signing the stream.\n"));
663 sd->signal(BNET_EOD); /* end of checksum */
665 if (ff_pkt->cmd_plugin) {
666 send_plugin_name(jcr, sd, false); /* signal end of plugin data */
670 rtnstat = 1; /* good return */
674 crypto_digest_free(digest);
676 if (signing_digest) {
677 crypto_digest_free(signing_digest);
680 crypto_sign_free(sig);
686 * Send data read from an already open file descriptor.
688 * We return 1 on sucess and 0 on errors.
691 * We use ff_pkt->statp.st_size when FO_SPARSE to know when to stop
693 * Currently this is not a problem as the only other stream, resource forks,
694 * are not handled as sparse files.
696 static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest,
697 DIGEST *signing_digest)
699 BSOCK *sd = jcr->store_bsock;
700 uint64_t fileAddr = 0; /* file address */
702 int32_t rsize = jcr->buf_size; /* read buffer size */
704 CIPHER_CONTEXT *cipher_ctx = NULL; /* Quell bogus uninitialized warnings */
705 const uint8_t *cipher_input;
706 uint32_t cipher_input_len;
707 uint32_t cipher_block_size;
708 uint32_t encrypted_len;
709 #ifdef FD_NO_SEND_TEST
714 rbuf = sd->msg; /* read buffer */
715 wbuf = sd->msg; /* write buffer */
716 cipher_input = (uint8_t *)rbuf; /* encrypt uncompressed data */
718 Dmsg1(300, "Saving data, type=%d\n", ff_pkt->type);
721 uLong compress_len = 0;
722 uLong max_compress_len = 0;
723 const Bytef *cbuf = NULL;
726 if (ff_pkt->flags & FO_GZIP) {
727 if (ff_pkt->flags & FO_SPARSE) {
728 cbuf = (Bytef *)jcr->compress_buf + SPARSE_FADDR_SIZE;
729 max_compress_len = jcr->compress_buf_size - SPARSE_FADDR_SIZE;
731 cbuf = (Bytef *)jcr->compress_buf;
732 max_compress_len = jcr->compress_buf_size; /* set max length */
734 wbuf = jcr->compress_buf; /* compressed output here */
735 cipher_input = (uint8_t *)jcr->compress_buf; /* encrypt compressed data */
738 * Only change zlib parameters if there is no pending operation.
739 * This should never happen as deflatereset is called after each
743 if (((z_stream*)jcr->pZLIB_compress_workset)->total_in == 0) {
744 /* set gzip compression level - must be done per file */
745 if ((zstat=deflateParams((z_stream*)jcr->pZLIB_compress_workset,
746 ff_pkt->GZIP_level, Z_DEFAULT_STRATEGY)) != Z_OK) {
747 Jmsg(jcr, M_FATAL, 0, _("Compression deflateParams error: %d\n"), zstat);
748 set_jcr_job_status(jcr, JS_ErrorTerminated);
754 const uint32_t max_compress_len = 0;
757 if (ff_pkt->flags & FO_ENCRYPT) {
758 if (ff_pkt->flags & FO_SPARSE) {
759 Jmsg0(jcr, M_FATAL, 0, _("Encrypting sparse data not supported.\n"));
762 /* Allocate the cipher context */
763 if ((cipher_ctx = crypto_cipher_new(jcr->crypto.pki_session, true,
764 &cipher_block_size)) == NULL) {
765 /* Shouldn't happen! */
766 Jmsg0(jcr, M_FATAL, 0, _("Failed to initialize encryption context.\n"));
771 * Grow the crypto buffer, if necessary.
772 * crypto_cipher_update() will buffer up to (cipher_block_size - 1).
773 * We grow crypto_buf to the maximum number of blocks that
774 * could be returned for the given read buffer size.
775 * (Using the larger of either rsize or max_compress_len)
777 jcr->crypto.crypto_buf = check_pool_memory_size(jcr->crypto.crypto_buf,
778 (MAX(rsize + (int)sizeof(uint32_t), (int32_t)max_compress_len) +
779 cipher_block_size - 1) / cipher_block_size * cipher_block_size);
781 wbuf = jcr->crypto.crypto_buf; /* Encrypted, possibly compressed output here. */
785 * Send Data header to Storage daemon
786 * <file-index> <stream> <info>
788 if (!sd->fsend("%ld %d 0", jcr->JobFiles, stream)) {
789 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
793 Dmsg1(300, ">stored: datahdr %s\n", sd->msg);
796 * Make space at beginning of buffer for fileAddr because this
797 * same buffer will be used for writing if compression is off.
799 if (ff_pkt->flags & FO_SPARSE) {
800 rbuf += SPARSE_FADDR_SIZE;
801 rsize -= SPARSE_FADDR_SIZE;
802 #ifdef HAVE_FREEBSD_OS
804 * To read FreeBSD partitions, the read size must be
807 rsize = (rsize/512) * 512;
811 /* a RAW device read on win32 only works if the buffer is a multiple of 512 */
813 if (S_ISBLK(ff_pkt->statp.st_mode))
814 rsize = (rsize/512) * 512;
820 while ((sd->msglen=(uint32_t)bread(&ff_pkt->bfd, rbuf, rsize)) > 0) {
822 /* Check for sparse blocks */
823 if (ff_pkt->flags & FO_SPARSE) {
825 bool allZeros = false;
826 if ((sd->msglen == rsize &&
827 fileAddr+sd->msglen < (uint64_t)ff_pkt->statp.st_size) ||
828 ((ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO) &&
829 (uint64_t)ff_pkt->statp.st_size == 0)) {
830 allZeros = is_buf_zero(rbuf, rsize);
833 /* Put file address as first data in buffer */
834 ser_begin(wbuf, SPARSE_FADDR_SIZE);
835 ser_uint64(fileAddr); /* store fileAddr in begin of buffer */
837 fileAddr += sd->msglen; /* update file address */
838 /* Skip block of all zeros */
840 continue; /* skip block of zeros */
844 jcr->ReadBytes += sd->msglen; /* count bytes read */
846 /* Uncompressed cipher input length */
847 cipher_input_len = sd->msglen;
849 /* Update checksum if requested */
851 crypto_digest_update(digest, (uint8_t *)rbuf, sd->msglen);
854 /* Update signing digest if requested */
855 if (signing_digest) {
856 crypto_digest_update(signing_digest, (uint8_t *)rbuf, sd->msglen);
860 /* Do compression if turned on */
861 if (ff_pkt->flags & FO_GZIP && jcr->pZLIB_compress_workset) {
862 Dmsg3(400, "cbuf=0x%x rbuf=0x%x len=%u\n", cbuf, rbuf, sd->msglen);
864 ((z_stream*)jcr->pZLIB_compress_workset)->next_in = (Bytef *)rbuf;
865 ((z_stream*)jcr->pZLIB_compress_workset)->avail_in = sd->msglen;
866 ((z_stream*)jcr->pZLIB_compress_workset)->next_out = (Bytef *)cbuf;
867 ((z_stream*)jcr->pZLIB_compress_workset)->avail_out = max_compress_len;
869 if ((zstat=deflate((z_stream*)jcr->pZLIB_compress_workset, Z_FINISH)) != Z_STREAM_END) {
870 Jmsg(jcr, M_FATAL, 0, _("Compression deflate error: %d\n"), zstat);
871 set_jcr_job_status(jcr, JS_ErrorTerminated);
874 compress_len = ((z_stream*)jcr->pZLIB_compress_workset)->total_out;
875 /* reset zlib stream to be able to begin from scratch again */
876 if ((zstat=deflateReset((z_stream*)jcr->pZLIB_compress_workset)) != Z_OK) {
877 Jmsg(jcr, M_FATAL, 0, _("Compression deflateReset error: %d\n"), zstat);
878 set_jcr_job_status(jcr, JS_ErrorTerminated);
882 Dmsg2(400, "compressed len=%d uncompressed len=%d\n", compress_len,
885 sd->msglen = compress_len; /* set compressed length */
886 cipher_input_len = compress_len;
890 * Note, here we prepend the current record length to the beginning
891 * of the encrypted data. This is because both sparse and compression
892 * restore handling want records returned to them with exactly the
893 * same number of bytes that were processed in the backup handling.
894 * That is, both are block filters rather than a stream. When doing
895 * compression, the compression routines may buffer data, so that for
896 * any one record compressed, when it is decompressed the same size
897 * will not be obtained. Of course, the buffered data eventually comes
898 * out in subsequent crypto_cipher_update() calls or at least
899 * when crypto_cipher_finalize() is called. Unfortunately, this
900 * "feature" of encryption enormously complicates the restore code.
902 if (ff_pkt->flags & FO_ENCRYPT) {
903 uint32_t initial_len = 0;
906 if (ff_pkt->flags & FO_SPARSE) {
907 cipher_input_len += SPARSE_FADDR_SIZE;
910 /* Encrypt the length of the input block */
911 uint8_t packet_len[sizeof(uint32_t)];
913 ser_begin(packet_len, sizeof(uint32_t));
914 ser_uint32(cipher_input_len); /* store data len in begin of buffer */
915 Dmsg1(20, "Encrypt len=%d\n", cipher_input_len);
917 if (!crypto_cipher_update(cipher_ctx, packet_len, sizeof(packet_len),
918 (uint8_t *)jcr->crypto.crypto_buf, &initial_len)) {
919 /* Encryption failed. Shouldn't happen. */
920 Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
924 /* Encrypt the input block */
925 if (crypto_cipher_update(cipher_ctx, cipher_input, cipher_input_len,
926 (uint8_t *)&jcr->crypto.crypto_buf[initial_len], &encrypted_len)) {
927 if ((initial_len + encrypted_len) == 0) {
928 /* No full block of data available, read more data */
931 Dmsg2(400, "encrypted len=%d unencrypted len=%d\n", encrypted_len,
933 sd->msglen = initial_len + encrypted_len; /* set encrypted length */
935 /* Encryption failed. Shouldn't happen. */
936 Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
941 /* Send the buffer to the Storage daemon */
942 if (ff_pkt->flags & FO_SPARSE) {
943 sd->msglen += SPARSE_FADDR_SIZE; /* include fileAddr in size */
945 sd->msg = wbuf; /* set correct write buffer */
947 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
951 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
953 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
954 sd->msg = msgsave; /* restore read buffer */
956 } /* end while read file data */
958 if (sd->msglen < 0) { /* error */
960 Jmsg(jcr, M_ERROR, 0, _("Read error on file %s. ERR=%s\n"),
961 ff_pkt->fname, be.bstrerror(ff_pkt->bfd.berrno));
962 if (jcr->JobErrors++ > 1000) { /* insanity check */
963 Jmsg(jcr, M_FATAL, 0, _("Too many errors.\n"));
965 } else if (ff_pkt->flags & FO_ENCRYPT) {
967 * For encryption, we must call finalize to push out any
970 if (!crypto_cipher_finalize(cipher_ctx, (uint8_t *)jcr->crypto.crypto_buf,
972 /* Padding failed. Shouldn't happen. */
973 Jmsg(jcr, M_FATAL, 0, _("Encryption padding error\n"));
977 /* Note, on SSL pre-0.9.7, there is always some output */
978 if (encrypted_len > 0) {
979 sd->msglen = encrypted_len; /* set encrypted length */
980 sd->msg = jcr->crypto.crypto_buf; /* set correct write buffer */
982 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
986 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
987 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
988 sd->msg = msgsave; /* restore bnet buffer */
992 if (!sd->signal(BNET_EOD)) { /* indicate end of file data */
993 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
998 /* Free the cipher context */
1000 crypto_cipher_free(cipher_ctx);
1005 /* Free the cipher context */
1007 crypto_cipher_free(cipher_ctx);
1010 sd->msg = msgsave; /* restore bnet buffer */
1015 bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream)
1017 BSOCK *sd = jcr->store_bsock;
1018 char attribs[MAXSTRING];
1019 char attribsEx[MAXSTRING];
1022 #ifdef FD_NO_SEND_TEST
1026 Dmsg1(300, "encode_and_send_attrs fname=%s\n", ff_pkt->fname);
1027 /* Find what data stream we will use, then encode the attributes */
1028 if ((data_stream = select_data_stream(ff_pkt)) == STREAM_NONE) {
1029 /* This should not happen */
1030 Jmsg0(jcr, M_FATAL, 0, _("Invalid file flags, no supported data stream type.\n"));
1033 encode_stat(attribs, &ff_pkt->statp, ff_pkt->LinkFI, data_stream);
1035 /* Now possibly extend the attributes */
1036 attr_stream = encode_attribsEx(jcr, attribsEx, ff_pkt);
1038 Dmsg3(300, "File %s\nattribs=%s\nattribsEx=%s\n", ff_pkt->fname, attribs, attribsEx);
1041 jcr->JobFiles++; /* increment number of files sent */
1042 ff_pkt->FileIndex = jcr->JobFiles; /* return FileIndex */
1043 pm_strcpy(jcr->last_fname, ff_pkt->fname);
1047 * Send Attributes header to Storage daemon
1048 * <file-index> <stream> <info>
1050 if (!sd->fsend("%ld %d 0", jcr->JobFiles, attr_stream)) {
1051 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1055 Dmsg1(300, ">stored: attrhdr %s\n", sd->msg);
1058 * Send file attributes to Storage daemon
1061 * Filename (full path)
1062 * Encoded attributes
1063 * Link name (if type==FT_LNK or FT_LNKSAVED)
1064 * Encoded extended-attributes (for Win32)
1066 * For a directory, link is the same as fname, but with trailing
1067 * slash. For a linked file, link is the link.
1069 if (ff_pkt->type != FT_DELETED) { /* already stripped */
1072 if (ff_pkt->type == FT_LNK || ff_pkt->type == FT_LNKSAVED) {
1073 Dmsg2(300, "Link %s to %s\n", ff_pkt->fname, ff_pkt->link);
1074 stat = sd->fsend("%ld %d %s%c%s%c%s%c%s%c", jcr->JobFiles,
1075 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, ff_pkt->link, 0,
1077 } else if (ff_pkt->type == FT_DIREND || ff_pkt->type == FT_REPARSE) {
1078 /* Here link is the canonical filename (i.e. with trailing slash) */
1079 stat = sd->fsend("%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
1080 ff_pkt->type, ff_pkt->link, 0, attribs, 0, 0, attribsEx, 0);
1082 stat = sd->fsend("%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
1083 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, 0, attribsEx, 0);
1085 if (ff_pkt->type != FT_DELETED) {
1086 unstrip_path(ff_pkt);
1089 Dmsg2(300, ">stored: attr len=%d: %s\n", sd->msglen, sd->msg);
1091 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1095 sd->signal(BNET_EOD); /* indicate end of attributes data */
1100 * Do in place strip of path
1102 static bool do_strip(int count, char *in)
1108 /* Copy to first path separator -- Win32 might have c: ... */
1109 while (*in && !IsPathSeparator(*in)) {
1113 numsep++; /* one separator seen */
1114 for (stripped=0; stripped<count && *in; stripped++) {
1115 while (*in && !IsPathSeparator(*in)) {
1116 in++; /* skip chars */
1119 numsep++; /* count separators seen */
1120 in++; /* skip separator */
1124 while (*in) { /* copy to end */
1125 if (IsPathSeparator(*in)) {
1131 Dmsg4(500, "stripped=%d count=%d numsep=%d sep>count=%d\n",
1132 stripped, count, numsep, numsep>count);
1133 return stripped==count && numsep>count;
1137 * If requested strip leading components of the path so that we can
1138 * save file as if it came from a subdirectory. This is most useful
1139 * for dealing with snapshots, by removing the snapshot directory, or
1140 * in handling vendor migrations where files have been restored with
1141 * a vendor product into a subdirectory.
1143 void strip_path(FF_PKT *ff_pkt)
1145 if (!(ff_pkt->flags & FO_STRIPPATH) || ff_pkt->strip_path <= 0) {
1146 Dmsg1(200, "No strip for %s\n", ff_pkt->fname);
1149 if (!ff_pkt->fname_save) {
1150 ff_pkt->fname_save = get_pool_memory(PM_FNAME);
1151 ff_pkt->link_save = get_pool_memory(PM_FNAME);
1153 pm_strcpy(ff_pkt->fname_save, ff_pkt->fname);
1154 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1155 pm_strcpy(ff_pkt->link_save, ff_pkt->link);
1156 Dmsg2(500, "strcpy link_save=%d link=%d\n", strlen(ff_pkt->link_save),
1157 strlen(ff_pkt->link));
1158 sm_check(__FILE__, __LINE__, true);
1162 * Strip path. If it doesn't succeed put it back. If
1163 * it does, and there is a different link string,
1164 * attempt to strip the link. If it fails, back them
1166 * Do not strip symlinks.
1167 * I.e. if either stripping fails don't strip anything.
1169 if (!do_strip(ff_pkt->strip_path, ff_pkt->fname)) {
1170 unstrip_path(ff_pkt);
1173 /* Strip links but not symlinks */
1174 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1175 if (!do_strip(ff_pkt->strip_path, ff_pkt->link)) {
1176 unstrip_path(ff_pkt);
1181 Dmsg3(100, "fname=%s stripped=%s link=%s\n", ff_pkt->fname_save, ff_pkt->fname,
1185 void unstrip_path(FF_PKT *ff_pkt)
1187 if (!(ff_pkt->flags & FO_STRIPPATH) || ff_pkt->strip_path <= 0) {
1190 strcpy(ff_pkt->fname, ff_pkt->fname_save);
1191 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1192 Dmsg2(500, "strcpy link=%s link_save=%s\n", ff_pkt->link,
1194 strcpy(ff_pkt->link, ff_pkt->link_save);
1195 Dmsg2(500, "strcpy link=%d link_save=%d\n", strlen(ff_pkt->link),
1196 strlen(ff_pkt->link_save));
1197 sm_check(__FILE__, __LINE__, true);