2 Bacula® - The Network Backup Solution
4 Copyright (C) 2000-2009 Free Software Foundation Europe e.V.
6 The main author of Bacula is Kern Sibbald, with contributions from
7 many others, a complete list can be found in the file AUTHORS.
8 This program is Free Software; you can redistribute it and/or
9 modify it under the terms of version two of the GNU General Public
10 License as published by the Free Software Foundation and included
13 This program is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
23 Bacula® is a registered trademark of Kern Sibbald.
24 The licensor of Bacula is the Free Software Foundation Europe
25 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
26 Switzerland, email:ftf@fsfeurope.org.
29 * Bacula File Daemon backup.c send file attributes and data
30 * to the Storage daemon.
32 * Kern Sibbald, March MM
41 /* Forward referenced functions */
42 int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level);
43 static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, DIGEST *signature_digest);
44 bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream);
45 static bool crypto_session_start(JCR *jcr);
46 static void crypto_session_end(JCR *jcr);
47 static bool crypto_session_send(JCR *jcr, BSOCK *sd);
50 * Find all the requested files and send them
51 * to the Storage daemon.
53 * Note, we normally carry on a one-way
54 * conversation from this point on with the SD, simply blasting
55 * data to him. To properly know what is going on, we
56 * also run a "heartbeat" monitor which reads the socket and
57 * reacts accordingly (at the moment it has nothing to do
58 * except echo the heartbeat to the Director).
61 bool blast_data_to_storage_daemon(JCR *jcr, char *addr)
65 // TODO landonf: Allow user to specify encryption algorithm
67 sd = jcr->store_bsock;
69 set_jcr_job_status(jcr, JS_Running);
71 Dmsg1(300, "bfiled: opened data connection %d to stored\n", sd->m_fd);
74 CLIENT *client = (CLIENT *)GetNextRes(R_CLIENT, NULL);
78 buf_size = client->max_network_buffer_size;
80 buf_size = 0; /* use default */
82 if (!sd->set_buffer_size(buf_size, BNET_SETBUF_WRITE)) {
83 set_jcr_job_status(jcr, JS_ErrorTerminated);
84 Jmsg(jcr, M_FATAL, 0, _("Cannot set buffer size FD->SD.\n"));
88 jcr->buf_size = sd->msglen;
89 /* Adjust for compression so that output buffer is
90 * 12 bytes + 0.1% larger than input buffer plus 18 bytes.
91 * This gives a bit extra plus room for the sparse addr if any.
92 * Note, we adjust the read size to be smaller so that the
93 * same output buffer can be used without growing it.
95 * The zlib compression workset is initialized here to minimize
96 * the "per file" load. The jcr member is only set, if the init
99 jcr->compress_buf_size = jcr->buf_size + ((jcr->buf_size+999) / 1000) + 30;
100 jcr->compress_buf = get_memory(jcr->compress_buf_size);
103 z_stream *pZlibStream = (z_stream*)malloc(sizeof(z_stream));
105 pZlibStream->zalloc = Z_NULL;
106 pZlibStream->zfree = Z_NULL;
107 pZlibStream->opaque = Z_NULL;
108 pZlibStream->state = Z_NULL;
110 if (deflateInit(pZlibStream, Z_DEFAULT_COMPRESSION) == Z_OK) {
111 jcr->pZLIB_compress_workset = pZlibStream;
118 if (!crypto_session_start(jcr)) {
122 set_find_options((FF_PKT *)jcr->ff, jcr->incremental, jcr->mtime);
124 /* in accurate mode, we overwrite the find_one check function */
126 set_find_changed_function((FF_PKT *)jcr->ff, accurate_check_file);
129 start_heartbeat_monitor(jcr);
131 jcr->acl_data = get_pool_memory(PM_MESSAGE);
132 jcr->xattr_data = get_pool_memory(PM_MESSAGE);
134 /* Subroutine save_file() is called for each file */
135 if (!find_files(jcr, (FF_PKT *)jcr->ff, save_file, plugin_save)) {
136 ok = false; /* error */
137 set_jcr_job_status(jcr, JS_ErrorTerminated);
140 accurate_send_deleted_list(jcr); /* send deleted list to SD */
142 stop_heartbeat_monitor(jcr);
144 sd->signal(BNET_EOD); /* end of sending data */
147 free_pool_memory(jcr->acl_data);
148 jcr->acl_data = NULL;
150 if (jcr->xattr_data) {
151 free_pool_memory(jcr->xattr_data);
152 jcr->xattr_data = NULL;
158 if (jcr->compress_buf) {
159 free_pool_memory(jcr->compress_buf);
160 jcr->compress_buf = NULL;
162 if (jcr->pZLIB_compress_workset) {
163 /* Free the zlib stream */
165 deflateEnd((z_stream *)jcr->pZLIB_compress_workset);
167 free (jcr->pZLIB_compress_workset);
168 jcr->pZLIB_compress_workset = NULL;
170 crypto_session_end(jcr);
173 Dmsg1(100, "end blast_data ok=%d\n", ok);
177 static bool crypto_session_start(JCR *jcr)
179 crypto_cipher_t cipher = CRYPTO_CIPHER_AES_128_CBC;
182 * Create encryption session data and a cached, DER-encoded session data
183 * structure. We use a single session key for each backup, so we'll encode
184 * the session data only once.
186 if (jcr->crypto.pki_encrypt) {
189 /* Create per-job session encryption context */
190 jcr->crypto.pki_session = crypto_session_new(cipher, jcr->crypto.pki_recipients);
192 /* Get the session data size */
193 if (!crypto_session_encode(jcr->crypto.pki_session, (uint8_t *)0, &size)) {
194 Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n"));
198 /* Allocate buffer */
199 jcr->crypto.pki_session_encoded = get_memory(size);
201 /* Encode session data */
202 if (!crypto_session_encode(jcr->crypto.pki_session, (uint8_t *)jcr->crypto.pki_session_encoded, &size)) {
203 Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n"));
207 /* ... and store the encoded size */
208 jcr->crypto.pki_session_encoded_size = size;
210 /* Allocate the encryption/decryption buffer */
211 jcr->crypto.crypto_buf = get_memory(CRYPTO_CIPHER_MAX_BLOCK_SIZE);
216 static void crypto_session_end(JCR *jcr)
218 if (jcr->crypto.crypto_buf) {
219 free_pool_memory(jcr->crypto.crypto_buf);
220 jcr->crypto.crypto_buf = NULL;
222 if (jcr->crypto.pki_session) {
223 crypto_session_free(jcr->crypto.pki_session);
225 if (jcr->crypto.pki_session_encoded) {
226 free_pool_memory(jcr->crypto.pki_session_encoded);
227 jcr->crypto.pki_session_encoded = NULL;
231 static bool crypto_session_send(JCR *jcr, BSOCK *sd)
235 /* Send our header */
236 Dmsg2(100, "Send hdr fi=%ld stream=%d\n", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA);
237 sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA);
240 sd->msg = jcr->crypto.pki_session_encoded;
241 sd->msglen = jcr->crypto.pki_session_encoded_size;
242 jcr->JobBytes += sd->msglen;
244 Dmsg1(100, "Send data len=%d\n", sd->msglen);
247 sd->signal(BNET_EOD);
253 * Called here by find() for each file included.
254 * This is a callback. The original is find_files() above.
256 * Send the file and its data to the Storage daemon.
260 * -1 to ignore file/directory (not used here)
262 int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level)
264 bool do_read = false;
265 int stat, data_stream;
267 DIGEST *digest = NULL;
268 DIGEST *signing_digest = NULL;
269 int digest_stream = STREAM_NONE;
270 SIGNATURE *sig = NULL;
271 bool has_file_data = false;
272 // TODO landonf: Allow the user to specify the digest algorithm
274 crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA256;
276 crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA1;
278 BSOCK *sd = jcr->store_bsock;
280 if (job_canceled(jcr)) {
284 jcr->num_files_examined++; /* bump total file count */
286 switch (ff_pkt->type) {
287 case FT_LNKSAVED: /* Hard linked, file already saved */
288 Dmsg2(130, "FT_LNKSAVED hard link: %s => %s\n", ff_pkt->fname, ff_pkt->link);
291 Dmsg1(130, "FT_REGE saving: %s\n", ff_pkt->fname);
292 has_file_data = true;
295 Dmsg1(130, "FT_REG saving: %s\n", ff_pkt->fname);
296 has_file_data = true;
299 Dmsg2(130, "FT_LNK saving: %s -> %s\n", ff_pkt->fname, ff_pkt->link);
302 jcr->num_files_examined--; /* correct file count */
303 return 1; /* not used */
305 Jmsg(jcr, M_INFO, 1, _(" Recursion turned off. Will not descend from %s into %s\n"),
306 ff_pkt->top_fname, ff_pkt->fname);
307 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
310 /* Suppress message for /dev filesystems */
311 if (!is_in_fileset(ff_pkt)) {
312 Jmsg(jcr, M_INFO, 1, _(" %s is a different filesystem. Will not descend from %s into %s\n"),
313 ff_pkt->fname, ff_pkt->top_fname, ff_pkt->fname);
315 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
318 Jmsg(jcr, M_INFO, 1, _(" Disallowed filesystem. Will not descend from %s into %s\n"),
319 ff_pkt->top_fname, ff_pkt->fname);
320 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
323 Jmsg(jcr, M_INFO, 1, _(" Disallowed drive type. Will not descend into %s\n"),
328 Dmsg1(130, "FT_DIREND: %s\n", ff_pkt->link);
331 Dmsg1(130, "FT_SPEC saving: %s\n", ff_pkt->fname);
332 if (S_ISSOCK(ff_pkt->statp.st_mode)) {
333 Jmsg(jcr, M_SKIPPED, 1, _(" Socket file skipped: %s\n"), ff_pkt->fname);
338 Dmsg1(130, "FT_RAW saving: %s\n", ff_pkt->fname);
339 has_file_data = true;
342 Dmsg1(130, "FT_FIFO saving: %s\n", ff_pkt->fname);
346 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not access \"%s\": ERR=%s\n"), ff_pkt->fname,
347 be.bstrerror(ff_pkt->ff_errno));
353 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not follow link \"%s\": ERR=%s\n"),
354 ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
360 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not stat \"%s\": ERR=%s\n"), ff_pkt->fname,
361 be.bstrerror(ff_pkt->ff_errno));
367 Jmsg(jcr, M_SKIPPED, 1, _(" Unchanged file skipped: %s\n"), ff_pkt->fname);
370 Jmsg(jcr, M_NOTSAVED, 0, _(" Archive file not saved: %s\n"), ff_pkt->fname);
374 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not open directory \"%s\": ERR=%s\n"),
375 ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
380 Jmsg(jcr, M_NOTSAVED, 0, _(" Unknown file type %d; not saved: %s\n"),
381 ff_pkt->type, ff_pkt->fname);
386 Dmsg1(130, "bfiled: sending %s to stored\n", ff_pkt->fname);
388 /* Digests and encryption are only useful if there's file data */
391 * Setup for digest handling. If this fails, the digest will be set to NULL
392 * and not used. Note, the digest (file hash) can be any one of the four
395 * The signing digest is a single algorithm depending on
396 * whether or not we have SHA2.
397 * ****FIXME**** the signing algoritm should really be
398 * determined a different way!!!!!! What happens if
399 * sha2 was available during backup but not restore?
401 if (ff_pkt->flags & FO_MD5) {
402 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_MD5);
403 digest_stream = STREAM_MD5_DIGEST;
405 } else if (ff_pkt->flags & FO_SHA1) {
406 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA1);
407 digest_stream = STREAM_SHA1_DIGEST;
409 } else if (ff_pkt->flags & FO_SHA256) {
410 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA256);
411 digest_stream = STREAM_SHA256_DIGEST;
413 } else if (ff_pkt->flags & FO_SHA512) {
414 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA512);
415 digest_stream = STREAM_SHA512_DIGEST;
418 /* Did digest initialization fail? */
419 if (digest_stream != STREAM_NONE && digest == NULL) {
420 Jmsg(jcr, M_WARNING, 0, _("%s digest initialization failed\n"),
421 stream_to_ascii(digest_stream));
425 * Set up signature digest handling. If this fails, the signature digest will be set to
428 // TODO landonf: We should really only calculate the digest once, for both verification and signing.
429 if (jcr->crypto.pki_sign) {
430 signing_digest = crypto_digest_new(jcr, signing_algorithm);
432 /* Full-stop if a failure occurred initializing the signature digest */
433 if (signing_digest == NULL) {
434 Jmsg(jcr, M_NOTSAVED, 0, _("%s signature digest initialization failed\n"),
435 stream_to_ascii(signing_algorithm));
441 /* Enable encryption */
442 if (jcr->crypto.pki_encrypt) {
443 ff_pkt->flags |= FO_ENCRYPT;
447 /* Initialize the file descriptor we use for data and other streams. */
449 if (ff_pkt->flags & FO_PORTABLE) {
450 set_portable_backup(&ff_pkt->bfd); /* disable Win32 BackupRead() */
452 if (ff_pkt->cmd_plugin) {
453 if (!set_cmd_plugin(&ff_pkt->bfd, jcr)) {
456 send_plugin_name(jcr, sd, true); /* signal start of plugin data */
459 /* Send attributes -- must be done after binit() */
460 if (!encode_and_send_attributes(jcr, ff_pkt, data_stream)) {
464 /* Set up the encryption context and send the session data to the SD */
465 if (has_file_data && jcr->crypto.pki_encrypt) {
466 if (!crypto_session_send(jcr, sd)) {
472 * Open any file with data that we intend to save, then save it.
474 * Note, if is_win32_backup, we must open the Directory so that
475 * the BackupRead will save its permissions and ownership streams.
477 if (ff_pkt->type != FT_LNKSAVED && S_ISREG(ff_pkt->statp.st_mode)) {
479 do_read = !is_portable_backup(&ff_pkt->bfd) || ff_pkt->statp.st_size > 0;
481 do_read = ff_pkt->statp.st_size > 0;
483 } else if (ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO ||
484 ff_pkt->type == FT_REPARSE ||
485 (!is_portable_backup(&ff_pkt->bfd) && ff_pkt->type == FT_DIREND)) {
488 if (ff_pkt->cmd_plugin) {
492 Dmsg1(400, "do_read=%d\n", do_read);
496 if (ff_pkt->type == FT_FIFO) {
497 tid = start_thread_timer(jcr, pthread_self(), 60);
501 int noatime = ff_pkt->flags & FO_NOATIME ? O_NOATIME : 0;
502 ff_pkt->bfd.reparse_point = ff_pkt->type == FT_REPARSE;
503 if (bopen(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY | noatime, 0) < 0) {
504 ff_pkt->ff_errno = errno;
506 Jmsg(jcr, M_NOTSAVED, 0, _(" Cannot open \"%s\": ERR=%s.\n"), ff_pkt->fname,
510 stop_thread_timer(tid);
516 stop_thread_timer(tid);
520 stat = send_data(jcr, data_stream, ff_pkt, digest, signing_digest);
522 if (ff_pkt->flags & FO_CHKCHANGES) {
523 has_file_changed(jcr, ff_pkt);
526 bclose(&ff_pkt->bfd);
533 #ifdef HAVE_DARWIN_OS
534 /* Regular files can have resource forks and Finder Info */
535 if (ff_pkt->type != FT_LNKSAVED && (S_ISREG(ff_pkt->statp.st_mode) &&
536 ff_pkt->flags & FO_HFSPLUS)) {
537 if (ff_pkt->hfsinfo.rsrclength > 0) {
540 if (!bopen_rsrc(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0) < 0) {
541 ff_pkt->ff_errno = errno;
543 Jmsg(jcr, M_NOTSAVED, -1, _(" Cannot open resource fork for \"%s\": ERR=%s.\n"),
544 ff_pkt->fname, be.bstrerror());
546 if (is_bopen(&ff_pkt->bfd)) {
547 bclose(&ff_pkt->bfd);
551 flags = ff_pkt->flags;
552 ff_pkt->flags &= ~(FO_GZIP|FO_SPARSE);
553 if (flags & FO_ENCRYPT) {
554 rsrc_stream = STREAM_ENCRYPTED_MACOS_FORK_DATA;
556 rsrc_stream = STREAM_MACOS_FORK_DATA;
558 stat = send_data(jcr, rsrc_stream, ff_pkt, digest, signing_digest);
559 ff_pkt->flags = flags;
560 bclose(&ff_pkt->bfd);
566 Dmsg1(300, "Saving Finder Info for \"%s\"\n", ff_pkt->fname);
567 sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_HFSPLUS_ATTRIBUTES);
568 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
569 pm_memcpy(sd->msg, ff_pkt->hfsinfo.fndrinfo, 32);
572 crypto_digest_update(digest, (uint8_t *)sd->msg, sd->msglen);
574 if (signing_digest) {
575 crypto_digest_update(signing_digest, (uint8_t *)sd->msg, sd->msglen);
578 sd->signal(BNET_EOD);
583 * Save ACLs for anything not being a symlink.
585 if (ff_pkt->flags & FO_ACL && ff_pkt->type != FT_LNK) {
586 if (!build_acl_streams(jcr, ff_pkt))
591 * Save Extended Attributes for all files.
593 if (ff_pkt->flags & FO_XATTR) {
594 if (!build_xattr_streams(jcr, ff_pkt))
598 /* Terminate the signing digest and send it to the Storage daemon */
599 if (signing_digest) {
602 if ((sig = crypto_sign_new(jcr)) == NULL) {
603 Jmsg(jcr, M_FATAL, 0, _("Failed to allocate memory for crypto signature.\n"));
607 if (!crypto_sign_add_signer(sig, signing_digest, jcr->crypto.pki_keypair)) {
608 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
612 /* Get signature size */
613 if (!crypto_sign_encode(sig, NULL, &size)) {
614 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
618 /* Grow the bsock buffer to fit our message if necessary */
619 if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
620 sd->msg = realloc_pool_memory(sd->msg, size);
623 /* Send our header */
624 sd->fsend("%ld %ld 0", jcr->JobFiles, STREAM_SIGNED_DIGEST);
625 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
627 /* Encode signature data */
628 if (!crypto_sign_encode(sig, (uint8_t *)sd->msg, &size)) {
629 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
635 sd->signal(BNET_EOD); /* end of checksum */
638 /* Terminate any digest and send it to Storage daemon */
642 sd->fsend("%ld %d 0", jcr->JobFiles, digest_stream);
643 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
645 size = CRYPTO_DIGEST_MAX_SIZE;
647 /* Grow the bsock buffer to fit our message if necessary */
648 if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
649 sd->msg = realloc_pool_memory(sd->msg, size);
652 if (!crypto_digest_finalize(digest, (uint8_t *)sd->msg, &size)) {
653 Jmsg(jcr, M_FATAL, 0, _("An error occurred finalizing signing the stream.\n"));
659 sd->signal(BNET_EOD); /* end of checksum */
661 if (ff_pkt->cmd_plugin) {
662 send_plugin_name(jcr, sd, false); /* signal end of plugin data */
666 rtnstat = 1; /* good return */
670 crypto_digest_free(digest);
672 if (signing_digest) {
673 crypto_digest_free(signing_digest);
676 crypto_sign_free(sig);
682 * Send data read from an already open file descriptor.
684 * We return 1 on sucess and 0 on errors.
687 * We use ff_pkt->statp.st_size when FO_SPARSE to know when to stop
689 * Currently this is not a problem as the only other stream, resource forks,
690 * are not handled as sparse files.
692 static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest,
693 DIGEST *signing_digest)
695 BSOCK *sd = jcr->store_bsock;
696 uint64_t fileAddr = 0; /* file address */
698 int32_t rsize = jcr->buf_size; /* read buffer size */
700 CIPHER_CONTEXT *cipher_ctx = NULL; /* Quell bogus uninitialized warnings */
701 const uint8_t *cipher_input;
702 uint32_t cipher_input_len;
703 uint32_t cipher_block_size;
704 uint32_t encrypted_len;
705 #ifdef FD_NO_SEND_TEST
710 rbuf = sd->msg; /* read buffer */
711 wbuf = sd->msg; /* write buffer */
712 cipher_input = (uint8_t *)rbuf; /* encrypt uncompressed data */
714 Dmsg1(300, "Saving data, type=%d\n", ff_pkt->type);
717 uLong compress_len = 0;
718 uLong max_compress_len = 0;
719 const Bytef *cbuf = NULL;
722 if (ff_pkt->flags & FO_GZIP) {
723 if (ff_pkt->flags & FO_SPARSE) {
724 cbuf = (Bytef *)jcr->compress_buf + SPARSE_FADDR_SIZE;
725 max_compress_len = jcr->compress_buf_size - SPARSE_FADDR_SIZE;
727 cbuf = (Bytef *)jcr->compress_buf;
728 max_compress_len = jcr->compress_buf_size; /* set max length */
730 wbuf = jcr->compress_buf; /* compressed output here */
731 cipher_input = (uint8_t *)jcr->compress_buf; /* encrypt compressed data */
734 * Only change zlib parameters if there is no pending operation.
735 * This should never happen as deflatereset is called after each
739 if (((z_stream*)jcr->pZLIB_compress_workset)->total_in == 0) {
740 /* set gzip compression level - must be done per file */
741 if ((zstat=deflateParams((z_stream*)jcr->pZLIB_compress_workset,
742 ff_pkt->GZIP_level, Z_DEFAULT_STRATEGY)) != Z_OK) {
743 Jmsg(jcr, M_FATAL, 0, _("Compression deflateParams error: %d\n"), zstat);
744 set_jcr_job_status(jcr, JS_ErrorTerminated);
750 const uint32_t max_compress_len = 0;
753 if (ff_pkt->flags & FO_ENCRYPT) {
754 if (ff_pkt->flags & FO_SPARSE) {
755 Jmsg0(jcr, M_FATAL, 0, _("Encrypting sparse data not supported.\n"));
758 /* Allocate the cipher context */
759 if ((cipher_ctx = crypto_cipher_new(jcr->crypto.pki_session, true,
760 &cipher_block_size)) == NULL) {
761 /* Shouldn't happen! */
762 Jmsg0(jcr, M_FATAL, 0, _("Failed to initialize encryption context.\n"));
767 * Grow the crypto buffer, if necessary.
768 * crypto_cipher_update() will buffer up to (cipher_block_size - 1).
769 * We grow crypto_buf to the maximum number of blocks that
770 * could be returned for the given read buffer size.
771 * (Using the larger of either rsize or max_compress_len)
773 jcr->crypto.crypto_buf = check_pool_memory_size(jcr->crypto.crypto_buf,
774 (MAX(rsize + (int)sizeof(uint32_t), (int32_t)max_compress_len) +
775 cipher_block_size - 1) / cipher_block_size * cipher_block_size);
777 wbuf = jcr->crypto.crypto_buf; /* Encrypted, possibly compressed output here. */
781 * Send Data header to Storage daemon
782 * <file-index> <stream> <info>
784 if (!sd->fsend("%ld %d 0", jcr->JobFiles, stream)) {
785 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
789 Dmsg1(300, ">stored: datahdr %s\n", sd->msg);
792 * Make space at beginning of buffer for fileAddr because this
793 * same buffer will be used for writing if compression is off.
795 if (ff_pkt->flags & FO_SPARSE) {
796 rbuf += SPARSE_FADDR_SIZE;
797 rsize -= SPARSE_FADDR_SIZE;
798 #ifdef HAVE_FREEBSD_OS
800 * To read FreeBSD partitions, the read size must be
803 rsize = (rsize/512) * 512;
807 /* a RAW device read on win32 only works if the buffer is a multiple of 512 */
809 if (S_ISBLK(ff_pkt->statp.st_mode))
810 rsize = (rsize/512) * 512;
816 while ((sd->msglen=(uint32_t)bread(&ff_pkt->bfd, rbuf, rsize)) > 0) {
818 /* Check for sparse blocks */
819 if (ff_pkt->flags & FO_SPARSE) {
821 bool allZeros = false;
822 if ((sd->msglen == rsize &&
823 fileAddr+sd->msglen < (uint64_t)ff_pkt->statp.st_size) ||
824 ((ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO) &&
825 (uint64_t)ff_pkt->statp.st_size == 0)) {
826 allZeros = is_buf_zero(rbuf, rsize);
829 /* Put file address as first data in buffer */
830 ser_begin(wbuf, SPARSE_FADDR_SIZE);
831 ser_uint64(fileAddr); /* store fileAddr in begin of buffer */
833 fileAddr += sd->msglen; /* update file address */
834 /* Skip block of all zeros */
836 continue; /* skip block of zeros */
840 jcr->ReadBytes += sd->msglen; /* count bytes read */
842 /* Uncompressed cipher input length */
843 cipher_input_len = sd->msglen;
845 /* Update checksum if requested */
847 crypto_digest_update(digest, (uint8_t *)rbuf, sd->msglen);
850 /* Update signing digest if requested */
851 if (signing_digest) {
852 crypto_digest_update(signing_digest, (uint8_t *)rbuf, sd->msglen);
856 /* Do compression if turned on */
857 if (ff_pkt->flags & FO_GZIP && jcr->pZLIB_compress_workset) {
858 Dmsg3(400, "cbuf=0x%x rbuf=0x%x len=%u\n", cbuf, rbuf, sd->msglen);
860 ((z_stream*)jcr->pZLIB_compress_workset)->next_in = (Bytef *)rbuf;
861 ((z_stream*)jcr->pZLIB_compress_workset)->avail_in = sd->msglen;
862 ((z_stream*)jcr->pZLIB_compress_workset)->next_out = (Bytef *)cbuf;
863 ((z_stream*)jcr->pZLIB_compress_workset)->avail_out = max_compress_len;
865 if ((zstat=deflate((z_stream*)jcr->pZLIB_compress_workset, Z_FINISH)) != Z_STREAM_END) {
866 Jmsg(jcr, M_FATAL, 0, _("Compression deflate error: %d\n"), zstat);
867 set_jcr_job_status(jcr, JS_ErrorTerminated);
870 compress_len = ((z_stream*)jcr->pZLIB_compress_workset)->total_out;
871 /* reset zlib stream to be able to begin from scratch again */
872 if ((zstat=deflateReset((z_stream*)jcr->pZLIB_compress_workset)) != Z_OK) {
873 Jmsg(jcr, M_FATAL, 0, _("Compression deflateReset error: %d\n"), zstat);
874 set_jcr_job_status(jcr, JS_ErrorTerminated);
878 Dmsg2(400, "compressed len=%d uncompressed len=%d\n", compress_len,
881 sd->msglen = compress_len; /* set compressed length */
882 cipher_input_len = compress_len;
886 * Note, here we prepend the current record length to the beginning
887 * of the encrypted data. This is because both sparse and compression
888 * restore handling want records returned to them with exactly the
889 * same number of bytes that were processed in the backup handling.
890 * That is, both are block filters rather than a stream. When doing
891 * compression, the compression routines may buffer data, so that for
892 * any one record compressed, when it is decompressed the same size
893 * will not be obtained. Of course, the buffered data eventually comes
894 * out in subsequent crypto_cipher_update() calls or at least
895 * when crypto_cipher_finalize() is called. Unfortunately, this
896 * "feature" of encryption enormously complicates the restore code.
898 if (ff_pkt->flags & FO_ENCRYPT) {
899 uint32_t initial_len = 0;
902 if (ff_pkt->flags & FO_SPARSE) {
903 cipher_input_len += SPARSE_FADDR_SIZE;
906 /* Encrypt the length of the input block */
907 uint8_t packet_len[sizeof(uint32_t)];
909 ser_begin(packet_len, sizeof(uint32_t));
910 ser_uint32(cipher_input_len); /* store data len in begin of buffer */
911 Dmsg1(20, "Encrypt len=%d\n", cipher_input_len);
913 if (!crypto_cipher_update(cipher_ctx, packet_len, sizeof(packet_len),
914 (uint8_t *)jcr->crypto.crypto_buf, &initial_len)) {
915 /* Encryption failed. Shouldn't happen. */
916 Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
920 /* Encrypt the input block */
921 if (crypto_cipher_update(cipher_ctx, cipher_input, cipher_input_len,
922 (uint8_t *)&jcr->crypto.crypto_buf[initial_len], &encrypted_len)) {
923 if ((initial_len + encrypted_len) == 0) {
924 /* No full block of data available, read more data */
927 Dmsg2(400, "encrypted len=%d unencrypted len=%d\n", encrypted_len,
929 sd->msglen = initial_len + encrypted_len; /* set encrypted length */
931 /* Encryption failed. Shouldn't happen. */
932 Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
937 /* Send the buffer to the Storage daemon */
938 if (ff_pkt->flags & FO_SPARSE) {
939 sd->msglen += SPARSE_FADDR_SIZE; /* include fileAddr in size */
941 sd->msg = wbuf; /* set correct write buffer */
943 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
947 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
949 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
950 sd->msg = msgsave; /* restore read buffer */
952 } /* end while read file data */
954 if (sd->msglen < 0) { /* error */
956 Jmsg(jcr, M_ERROR, 0, _("Read error on file %s. ERR=%s\n"),
957 ff_pkt->fname, be.bstrerror(ff_pkt->bfd.berrno));
958 if (jcr->JobErrors++ > 1000) { /* insanity check */
959 Jmsg(jcr, M_FATAL, 0, _("Too many errors.\n"));
961 } else if (ff_pkt->flags & FO_ENCRYPT) {
963 * For encryption, we must call finalize to push out any
966 if (!crypto_cipher_finalize(cipher_ctx, (uint8_t *)jcr->crypto.crypto_buf,
968 /* Padding failed. Shouldn't happen. */
969 Jmsg(jcr, M_FATAL, 0, _("Encryption padding error\n"));
973 /* Note, on SSL pre-0.9.7, there is always some output */
974 if (encrypted_len > 0) {
975 sd->msglen = encrypted_len; /* set encrypted length */
976 sd->msg = jcr->crypto.crypto_buf; /* set correct write buffer */
978 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
982 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
983 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
984 sd->msg = msgsave; /* restore bnet buffer */
988 if (!sd->signal(BNET_EOD)) { /* indicate end of file data */
989 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
994 /* Free the cipher context */
996 crypto_cipher_free(cipher_ctx);
1001 /* Free the cipher context */
1003 crypto_cipher_free(cipher_ctx);
1006 sd->msg = msgsave; /* restore bnet buffer */
1011 bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream)
1013 BSOCK *sd = jcr->store_bsock;
1014 char attribs[MAXSTRING];
1015 char attribsEx[MAXSTRING];
1018 #ifdef FD_NO_SEND_TEST
1022 Dmsg1(300, "encode_and_send_attrs fname=%s\n", ff_pkt->fname);
1023 /* Find what data stream we will use, then encode the attributes */
1024 if ((data_stream = select_data_stream(ff_pkt)) == STREAM_NONE) {
1025 /* This should not happen */
1026 Jmsg0(jcr, M_FATAL, 0, _("Invalid file flags, no supported data stream type.\n"));
1029 encode_stat(attribs, &ff_pkt->statp, ff_pkt->LinkFI, data_stream);
1031 /* Now possibly extend the attributes */
1032 attr_stream = encode_attribsEx(jcr, attribsEx, ff_pkt);
1034 Dmsg3(300, "File %s\nattribs=%s\nattribsEx=%s\n", ff_pkt->fname, attribs, attribsEx);
1037 jcr->JobFiles++; /* increment number of files sent */
1038 ff_pkt->FileIndex = jcr->JobFiles; /* return FileIndex */
1039 pm_strcpy(jcr->last_fname, ff_pkt->fname);
1043 * Send Attributes header to Storage daemon
1044 * <file-index> <stream> <info>
1046 if (!sd->fsend("%ld %d 0", jcr->JobFiles, attr_stream)) {
1047 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1051 Dmsg1(300, ">stored: attrhdr %s\n", sd->msg);
1054 * Send file attributes to Storage daemon
1057 * Filename (full path)
1058 * Encoded attributes
1059 * Link name (if type==FT_LNK or FT_LNKSAVED)
1060 * Encoded extended-attributes (for Win32)
1062 * For a directory, link is the same as fname, but with trailing
1063 * slash. For a linked file, link is the link.
1065 if (ff_pkt->type != FT_DELETED) { /* already stripped */
1068 if (ff_pkt->type == FT_LNK || ff_pkt->type == FT_LNKSAVED) {
1069 Dmsg2(300, "Link %s to %s\n", ff_pkt->fname, ff_pkt->link);
1070 stat = sd->fsend("%ld %d %s%c%s%c%s%c%s%c", jcr->JobFiles,
1071 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, ff_pkt->link, 0,
1073 } else if (ff_pkt->type == FT_DIREND || ff_pkt->type == FT_REPARSE) {
1074 /* Here link is the canonical filename (i.e. with trailing slash) */
1075 stat = sd->fsend("%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
1076 ff_pkt->type, ff_pkt->link, 0, attribs, 0, 0, attribsEx, 0);
1078 stat = sd->fsend("%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
1079 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, 0, attribsEx, 0);
1081 if (ff_pkt->type != FT_DELETED) {
1082 unstrip_path(ff_pkt);
1085 Dmsg2(300, ">stored: attr len=%d: %s\n", sd->msglen, sd->msg);
1087 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1091 sd->signal(BNET_EOD); /* indicate end of attributes data */
1096 * Do in place strip of path
1098 static bool do_strip(int count, char *in)
1104 /* Copy to first path separator -- Win32 might have c: ... */
1105 while (*in && !IsPathSeparator(*in)) {
1109 numsep++; /* one separator seen */
1110 for (stripped=0; stripped<count && *in; stripped++) {
1111 while (*in && !IsPathSeparator(*in)) {
1112 in++; /* skip chars */
1115 numsep++; /* count separators seen */
1116 in++; /* skip separator */
1120 while (*in) { /* copy to end */
1121 if (IsPathSeparator(*in)) {
1127 Dmsg4(500, "stripped=%d count=%d numsep=%d sep>count=%d\n",
1128 stripped, count, numsep, numsep>count);
1129 return stripped==count && numsep>count;
1133 * If requested strip leading components of the path so that we can
1134 * save file as if it came from a subdirectory. This is most useful
1135 * for dealing with snapshots, by removing the snapshot directory, or
1136 * in handling vendor migrations where files have been restored with
1137 * a vendor product into a subdirectory.
1139 void strip_path(FF_PKT *ff_pkt)
1141 if (!(ff_pkt->flags & FO_STRIPPATH) || ff_pkt->strip_path <= 0) {
1142 Dmsg1(200, "No strip for %s\n", ff_pkt->fname);
1145 if (!ff_pkt->fname_save) {
1146 ff_pkt->fname_save = get_pool_memory(PM_FNAME);
1147 ff_pkt->link_save = get_pool_memory(PM_FNAME);
1149 pm_strcpy(ff_pkt->fname_save, ff_pkt->fname);
1150 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1151 pm_strcpy(ff_pkt->link_save, ff_pkt->link);
1152 Dmsg2(500, "strcpy link_save=%d link=%d\n", strlen(ff_pkt->link_save),
1153 strlen(ff_pkt->link));
1154 sm_check(__FILE__, __LINE__, true);
1158 * Strip path. If it doesn't succeed put it back. If
1159 * it does, and there is a different link string,
1160 * attempt to strip the link. If it fails, back them
1162 * Do not strip symlinks.
1163 * I.e. if either stripping fails don't strip anything.
1165 if (!do_strip(ff_pkt->strip_path, ff_pkt->fname)) {
1166 unstrip_path(ff_pkt);
1169 /* Strip links but not symlinks */
1170 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1171 if (!do_strip(ff_pkt->strip_path, ff_pkt->link)) {
1172 unstrip_path(ff_pkt);
1177 Dmsg3(100, "fname=%s stripped=%s link=%s\n", ff_pkt->fname_save, ff_pkt->fname,
1181 void unstrip_path(FF_PKT *ff_pkt)
1183 if (!(ff_pkt->flags & FO_STRIPPATH) || ff_pkt->strip_path <= 0) {
1186 strcpy(ff_pkt->fname, ff_pkt->fname_save);
1187 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1188 Dmsg2(500, "strcpy link=%s link_save=%s\n", ff_pkt->link,
1190 strcpy(ff_pkt->link, ff_pkt->link_save);
1191 Dmsg2(500, "strcpy link=%d link_save=%d\n", strlen(ff_pkt->link),
1192 strlen(ff_pkt->link_save));
1193 sm_check(__FILE__, __LINE__, true);