2 Bacula® - The Network Backup Solution
4 Copyright (C) 2000-2009 Free Software Foundation Europe e.V.
6 The main author of Bacula is Kern Sibbald, with contributions from
7 many others, a complete list can be found in the file AUTHORS.
8 This program is Free Software; you can redistribute it and/or
9 modify it under the terms of version two of the GNU General Public
10 License as published by the Free Software Foundation and included
13 This program is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
23 Bacula® is a registered trademark of Kern Sibbald.
24 The licensor of Bacula is the Free Software Foundation Europe
25 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
26 Switzerland, email:ftf@fsfeurope.org.
29 * Bacula File Daemon backup.c send file attributes and data
30 * to the Storage daemon.
32 * Kern Sibbald, March MM
42 const bool have_acl = true;
44 const bool have_acl = false;
47 #if defined(HAVE_XATTR)
48 const bool have_xattr = true;
50 const bool have_xattr = false;
53 /* Forward referenced functions */
54 int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level);
55 static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, DIGEST *signature_digest);
56 bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream);
57 static bool crypto_session_start(JCR *jcr);
58 static void crypto_session_end(JCR *jcr);
59 static bool crypto_session_send(JCR *jcr, BSOCK *sd);
62 * Find all the requested files and send them
63 * to the Storage daemon.
65 * Note, we normally carry on a one-way
66 * conversation from this point on with the SD, simply blasting
67 * data to him. To properly know what is going on, we
68 * also run a "heartbeat" monitor which reads the socket and
69 * reacts accordingly (at the moment it has nothing to do
70 * except echo the heartbeat to the Director).
73 bool blast_data_to_storage_daemon(JCR *jcr, char *addr)
77 // TODO landonf: Allow user to specify encryption algorithm
79 sd = jcr->store_bsock;
81 set_jcr_job_status(jcr, JS_Running);
83 Dmsg1(300, "bfiled: opened data connection %d to stored\n", sd->m_fd);
86 CLIENT *client = (CLIENT *)GetNextRes(R_CLIENT, NULL);
90 buf_size = client->max_network_buffer_size;
92 buf_size = 0; /* use default */
94 if (!sd->set_buffer_size(buf_size, BNET_SETBUF_WRITE)) {
95 set_jcr_job_status(jcr, JS_ErrorTerminated);
96 Jmsg(jcr, M_FATAL, 0, _("Cannot set buffer size FD->SD.\n"));
100 jcr->buf_size = sd->msglen;
101 /* Adjust for compression so that output buffer is
102 * 12 bytes + 0.1% larger than input buffer plus 18 bytes.
103 * This gives a bit extra plus room for the sparse addr if any.
104 * Note, we adjust the read size to be smaller so that the
105 * same output buffer can be used without growing it.
107 * The zlib compression workset is initialized here to minimize
108 * the "per file" load. The jcr member is only set, if the init
111 jcr->compress_buf_size = jcr->buf_size + ((jcr->buf_size+999) / 1000) + 30;
112 jcr->compress_buf = get_memory(jcr->compress_buf_size);
115 z_stream *pZlibStream = (z_stream*)malloc(sizeof(z_stream));
117 pZlibStream->zalloc = Z_NULL;
118 pZlibStream->zfree = Z_NULL;
119 pZlibStream->opaque = Z_NULL;
120 pZlibStream->state = Z_NULL;
122 if (deflateInit(pZlibStream, Z_DEFAULT_COMPRESSION) == Z_OK) {
123 jcr->pZLIB_compress_workset = pZlibStream;
130 if (!crypto_session_start(jcr)) {
134 set_find_options((FF_PKT *)jcr->ff, jcr->incremental, jcr->mtime);
136 /* in accurate mode, we overwrite the find_one check function */
138 set_find_changed_function((FF_PKT *)jcr->ff, accurate_check_file);
141 start_heartbeat_monitor(jcr);
144 jcr->acl_data = get_pool_memory(PM_MESSAGE);
147 jcr->xattr_data = get_pool_memory(PM_MESSAGE);
150 /* Subroutine save_file() is called for each file */
151 if (!find_files(jcr, (FF_PKT *)jcr->ff, save_file, plugin_save)) {
152 ok = false; /* error */
153 set_jcr_job_status(jcr, JS_ErrorTerminated);
156 accurate_send_deleted_list(jcr); /* send deleted list to SD */
158 stop_heartbeat_monitor(jcr);
160 sd->signal(BNET_EOD); /* end of sending data */
162 if (have_acl && jcr->acl_data) {
163 free_pool_memory(jcr->acl_data);
164 jcr->acl_data = NULL;
166 if (have_xattr && jcr->xattr_data) {
167 free_pool_memory(jcr->xattr_data);
168 jcr->xattr_data = NULL;
174 if (jcr->compress_buf) {
175 free_pool_memory(jcr->compress_buf);
176 jcr->compress_buf = NULL;
178 if (jcr->pZLIB_compress_workset) {
179 /* Free the zlib stream */
181 deflateEnd((z_stream *)jcr->pZLIB_compress_workset);
183 free (jcr->pZLIB_compress_workset);
184 jcr->pZLIB_compress_workset = NULL;
186 crypto_session_end(jcr);
189 Dmsg1(100, "end blast_data ok=%d\n", ok);
193 static bool crypto_session_start(JCR *jcr)
195 crypto_cipher_t cipher = CRYPTO_CIPHER_AES_128_CBC;
198 * Create encryption session data and a cached, DER-encoded session data
199 * structure. We use a single session key for each backup, so we'll encode
200 * the session data only once.
202 if (jcr->crypto.pki_encrypt) {
205 /* Create per-job session encryption context */
206 jcr->crypto.pki_session = crypto_session_new(cipher, jcr->crypto.pki_recipients);
208 /* Get the session data size */
209 if (!crypto_session_encode(jcr->crypto.pki_session, (uint8_t *)0, &size)) {
210 Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n"));
214 /* Allocate buffer */
215 jcr->crypto.pki_session_encoded = get_memory(size);
217 /* Encode session data */
218 if (!crypto_session_encode(jcr->crypto.pki_session, (uint8_t *)jcr->crypto.pki_session_encoded, &size)) {
219 Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n"));
223 /* ... and store the encoded size */
224 jcr->crypto.pki_session_encoded_size = size;
226 /* Allocate the encryption/decryption buffer */
227 jcr->crypto.crypto_buf = get_memory(CRYPTO_CIPHER_MAX_BLOCK_SIZE);
232 static void crypto_session_end(JCR *jcr)
234 if (jcr->crypto.crypto_buf) {
235 free_pool_memory(jcr->crypto.crypto_buf);
236 jcr->crypto.crypto_buf = NULL;
238 if (jcr->crypto.pki_session) {
239 crypto_session_free(jcr->crypto.pki_session);
241 if (jcr->crypto.pki_session_encoded) {
242 free_pool_memory(jcr->crypto.pki_session_encoded);
243 jcr->crypto.pki_session_encoded = NULL;
247 static bool crypto_session_send(JCR *jcr, BSOCK *sd)
251 /* Send our header */
252 Dmsg2(100, "Send hdr fi=%ld stream=%d\n", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA);
253 sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA);
256 sd->msg = jcr->crypto.pki_session_encoded;
257 sd->msglen = jcr->crypto.pki_session_encoded_size;
258 jcr->JobBytes += sd->msglen;
260 Dmsg1(100, "Send data len=%d\n", sd->msglen);
263 sd->signal(BNET_EOD);
269 * Called here by find() for each file included.
270 * This is a callback. The original is find_files() above.
272 * Send the file and its data to the Storage daemon.
276 * -1 to ignore file/directory (not used here)
278 int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level)
280 bool do_read = false;
281 int stat, data_stream;
283 DIGEST *digest = NULL;
284 DIGEST *signing_digest = NULL;
285 int digest_stream = STREAM_NONE;
286 SIGNATURE *sig = NULL;
287 bool has_file_data = false;
288 // TODO landonf: Allow the user to specify the digest algorithm
290 crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA256;
292 crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA1;
294 BSOCK *sd = jcr->store_bsock;
296 if (job_canceled(jcr)) {
300 jcr->num_files_examined++; /* bump total file count */
302 switch (ff_pkt->type) {
303 case FT_LNKSAVED: /* Hard linked, file already saved */
304 Dmsg2(130, "FT_LNKSAVED hard link: %s => %s\n", ff_pkt->fname, ff_pkt->link);
307 Dmsg1(130, "FT_REGE saving: %s\n", ff_pkt->fname);
308 has_file_data = true;
311 Dmsg1(130, "FT_REG saving: %s\n", ff_pkt->fname);
312 has_file_data = true;
315 Dmsg2(130, "FT_LNK saving: %s -> %s\n", ff_pkt->fname, ff_pkt->link);
318 jcr->num_files_examined--; /* correct file count */
319 return 1; /* not used */
321 Jmsg(jcr, M_INFO, 1, _(" Recursion turned off. Will not descend from %s into %s\n"),
322 ff_pkt->top_fname, ff_pkt->fname);
323 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
326 /* Suppress message for /dev filesystems */
327 if (!is_in_fileset(ff_pkt)) {
328 Jmsg(jcr, M_INFO, 1, _(" %s is a different filesystem. Will not descend from %s into %s\n"),
329 ff_pkt->fname, ff_pkt->top_fname, ff_pkt->fname);
331 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
334 Jmsg(jcr, M_INFO, 1, _(" Disallowed filesystem. Will not descend from %s into %s\n"),
335 ff_pkt->top_fname, ff_pkt->fname);
336 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
339 Jmsg(jcr, M_INFO, 1, _(" Disallowed drive type. Will not descend into %s\n"),
344 Dmsg1(130, "FT_DIREND: %s\n", ff_pkt->link);
347 Dmsg1(130, "FT_SPEC saving: %s\n", ff_pkt->fname);
348 if (S_ISSOCK(ff_pkt->statp.st_mode)) {
349 Jmsg(jcr, M_SKIPPED, 1, _(" Socket file skipped: %s\n"), ff_pkt->fname);
354 Dmsg1(130, "FT_RAW saving: %s\n", ff_pkt->fname);
355 has_file_data = true;
358 Dmsg1(130, "FT_FIFO saving: %s\n", ff_pkt->fname);
362 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not access \"%s\": ERR=%s\n"), ff_pkt->fname,
363 be.bstrerror(ff_pkt->ff_errno));
369 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not follow link \"%s\": ERR=%s\n"),
370 ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
376 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not stat \"%s\": ERR=%s\n"), ff_pkt->fname,
377 be.bstrerror(ff_pkt->ff_errno));
383 Jmsg(jcr, M_SKIPPED, 1, _(" Unchanged file skipped: %s\n"), ff_pkt->fname);
386 Jmsg(jcr, M_NOTSAVED, 0, _(" Archive file not saved: %s\n"), ff_pkt->fname);
390 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not open directory \"%s\": ERR=%s\n"),
391 ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
396 Jmsg(jcr, M_NOTSAVED, 0, _(" Unknown file type %d; not saved: %s\n"),
397 ff_pkt->type, ff_pkt->fname);
402 Dmsg1(130, "bfiled: sending %s to stored\n", ff_pkt->fname);
404 /* Digests and encryption are only useful if there's file data */
407 * Setup for digest handling. If this fails, the digest will be set to NULL
408 * and not used. Note, the digest (file hash) can be any one of the four
411 * The signing digest is a single algorithm depending on
412 * whether or not we have SHA2.
413 * ****FIXME**** the signing algoritm should really be
414 * determined a different way!!!!!! What happens if
415 * sha2 was available during backup but not restore?
417 if (ff_pkt->flags & FO_MD5) {
418 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_MD5);
419 digest_stream = STREAM_MD5_DIGEST;
421 } else if (ff_pkt->flags & FO_SHA1) {
422 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA1);
423 digest_stream = STREAM_SHA1_DIGEST;
425 } else if (ff_pkt->flags & FO_SHA256) {
426 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA256);
427 digest_stream = STREAM_SHA256_DIGEST;
429 } else if (ff_pkt->flags & FO_SHA512) {
430 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA512);
431 digest_stream = STREAM_SHA512_DIGEST;
434 /* Did digest initialization fail? */
435 if (digest_stream != STREAM_NONE && digest == NULL) {
436 Jmsg(jcr, M_WARNING, 0, _("%s digest initialization failed\n"),
437 stream_to_ascii(digest_stream));
441 * Set up signature digest handling. If this fails, the signature digest will be set to
444 // TODO landonf: We should really only calculate the digest once, for both verification and signing.
445 if (jcr->crypto.pki_sign) {
446 signing_digest = crypto_digest_new(jcr, signing_algorithm);
448 /* Full-stop if a failure occurred initializing the signature digest */
449 if (signing_digest == NULL) {
450 Jmsg(jcr, M_NOTSAVED, 0, _("%s signature digest initialization failed\n"),
451 stream_to_ascii(signing_algorithm));
457 /* Enable encryption */
458 if (jcr->crypto.pki_encrypt) {
459 ff_pkt->flags |= FO_ENCRYPT;
463 /* Initialize the file descriptor we use for data and other streams. */
465 if (ff_pkt->flags & FO_PORTABLE) {
466 set_portable_backup(&ff_pkt->bfd); /* disable Win32 BackupRead() */
468 if (ff_pkt->cmd_plugin) {
469 if (!set_cmd_plugin(&ff_pkt->bfd, jcr)) {
472 send_plugin_name(jcr, sd, true); /* signal start of plugin data */
475 /* Send attributes -- must be done after binit() */
476 if (!encode_and_send_attributes(jcr, ff_pkt, data_stream)) {
480 /* Set up the encryption context and send the session data to the SD */
481 if (has_file_data && jcr->crypto.pki_encrypt) {
482 if (!crypto_session_send(jcr, sd)) {
488 * Open any file with data that we intend to save, then save it.
490 * Note, if is_win32_backup, we must open the Directory so that
491 * the BackupRead will save its permissions and ownership streams.
493 if (ff_pkt->type != FT_LNKSAVED && S_ISREG(ff_pkt->statp.st_mode)) {
495 do_read = !is_portable_backup(&ff_pkt->bfd) || ff_pkt->statp.st_size > 0;
497 do_read = ff_pkt->statp.st_size > 0;
499 } else if (ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO ||
500 ff_pkt->type == FT_REPARSE ||
501 (!is_portable_backup(&ff_pkt->bfd) && ff_pkt->type == FT_DIREND)) {
504 if (ff_pkt->cmd_plugin) {
508 Dmsg1(400, "do_read=%d\n", do_read);
512 if (ff_pkt->type == FT_FIFO) {
513 tid = start_thread_timer(jcr, pthread_self(), 60);
517 int noatime = ff_pkt->flags & FO_NOATIME ? O_NOATIME : 0;
518 ff_pkt->bfd.reparse_point = ff_pkt->type == FT_REPARSE;
519 if (bopen(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY | noatime, 0) < 0) {
520 ff_pkt->ff_errno = errno;
522 Jmsg(jcr, M_NOTSAVED, 0, _(" Cannot open \"%s\": ERR=%s.\n"), ff_pkt->fname,
526 stop_thread_timer(tid);
532 stop_thread_timer(tid);
536 stat = send_data(jcr, data_stream, ff_pkt, digest, signing_digest);
538 if (ff_pkt->flags & FO_CHKCHANGES) {
539 has_file_changed(jcr, ff_pkt);
542 bclose(&ff_pkt->bfd);
549 #ifdef HAVE_DARWIN_OS
550 /* Regular files can have resource forks and Finder Info */
551 if (ff_pkt->type != FT_LNKSAVED && (S_ISREG(ff_pkt->statp.st_mode) &&
552 ff_pkt->flags & FO_HFSPLUS)) {
553 if (ff_pkt->hfsinfo.rsrclength > 0) {
556 if (!bopen_rsrc(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0) < 0) {
557 ff_pkt->ff_errno = errno;
559 Jmsg(jcr, M_NOTSAVED, -1, _(" Cannot open resource fork for \"%s\": ERR=%s.\n"),
560 ff_pkt->fname, be.bstrerror());
562 if (is_bopen(&ff_pkt->bfd)) {
563 bclose(&ff_pkt->bfd);
567 flags = ff_pkt->flags;
568 ff_pkt->flags &= ~(FO_GZIP|FO_SPARSE);
569 if (flags & FO_ENCRYPT) {
570 rsrc_stream = STREAM_ENCRYPTED_MACOS_FORK_DATA;
572 rsrc_stream = STREAM_MACOS_FORK_DATA;
574 stat = send_data(jcr, rsrc_stream, ff_pkt, digest, signing_digest);
575 ff_pkt->flags = flags;
576 bclose(&ff_pkt->bfd);
582 Dmsg1(300, "Saving Finder Info for \"%s\"\n", ff_pkt->fname);
583 sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_HFSPLUS_ATTRIBUTES);
584 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
585 pm_memcpy(sd->msg, ff_pkt->hfsinfo.fndrinfo, 32);
588 crypto_digest_update(digest, (uint8_t *)sd->msg, sd->msglen);
590 if (signing_digest) {
591 crypto_digest_update(signing_digest, (uint8_t *)sd->msg, sd->msglen);
594 sd->signal(BNET_EOD);
599 * Save ACLs when requested and available for anything not being a symlink and not being a plugin.
601 if (ff_pkt->flags & FO_ACL && have_acl &&
602 ff_pkt->type != FT_LNK && !ff_pkt->cmd_plugin) {
603 if (!build_acl_streams(jcr, ff_pkt))
608 * Save Extended Attributes when requested and available for all files not being a plugin.
610 if (ff_pkt->flags & FO_XATTR && have_xattr &&
611 !ff_pkt->cmd_plugin) {
612 if (!build_xattr_streams(jcr, ff_pkt))
616 /* Terminate the signing digest and send it to the Storage daemon */
617 if (signing_digest) {
620 if ((sig = crypto_sign_new(jcr)) == NULL) {
621 Jmsg(jcr, M_FATAL, 0, _("Failed to allocate memory for crypto signature.\n"));
625 if (!crypto_sign_add_signer(sig, signing_digest, jcr->crypto.pki_keypair)) {
626 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
630 /* Get signature size */
631 if (!crypto_sign_encode(sig, NULL, &size)) {
632 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
636 /* Grow the bsock buffer to fit our message if necessary */
637 if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
638 sd->msg = realloc_pool_memory(sd->msg, size);
641 /* Send our header */
642 sd->fsend("%ld %ld 0", jcr->JobFiles, STREAM_SIGNED_DIGEST);
643 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
645 /* Encode signature data */
646 if (!crypto_sign_encode(sig, (uint8_t *)sd->msg, &size)) {
647 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
653 sd->signal(BNET_EOD); /* end of checksum */
656 /* Terminate any digest and send it to Storage daemon */
660 sd->fsend("%ld %d 0", jcr->JobFiles, digest_stream);
661 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
663 size = CRYPTO_DIGEST_MAX_SIZE;
665 /* Grow the bsock buffer to fit our message if necessary */
666 if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
667 sd->msg = realloc_pool_memory(sd->msg, size);
670 if (!crypto_digest_finalize(digest, (uint8_t *)sd->msg, &size)) {
671 Jmsg(jcr, M_FATAL, 0, _("An error occurred finalizing signing the stream.\n"));
677 sd->signal(BNET_EOD); /* end of checksum */
679 if (ff_pkt->cmd_plugin) {
680 send_plugin_name(jcr, sd, false); /* signal end of plugin data */
684 rtnstat = 1; /* good return */
688 crypto_digest_free(digest);
690 if (signing_digest) {
691 crypto_digest_free(signing_digest);
694 crypto_sign_free(sig);
700 * Send data read from an already open file descriptor.
702 * We return 1 on sucess and 0 on errors.
705 * We use ff_pkt->statp.st_size when FO_SPARSE to know when to stop
707 * Currently this is not a problem as the only other stream, resource forks,
708 * are not handled as sparse files.
710 static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest,
711 DIGEST *signing_digest)
713 BSOCK *sd = jcr->store_bsock;
714 uint64_t fileAddr = 0; /* file address */
716 int32_t rsize = jcr->buf_size; /* read buffer size */
718 CIPHER_CONTEXT *cipher_ctx = NULL; /* Quell bogus uninitialized warnings */
719 const uint8_t *cipher_input;
720 uint32_t cipher_input_len;
721 uint32_t cipher_block_size;
722 uint32_t encrypted_len;
723 #ifdef FD_NO_SEND_TEST
728 rbuf = sd->msg; /* read buffer */
729 wbuf = sd->msg; /* write buffer */
730 cipher_input = (uint8_t *)rbuf; /* encrypt uncompressed data */
732 Dmsg1(300, "Saving data, type=%d\n", ff_pkt->type);
735 uLong compress_len = 0;
736 uLong max_compress_len = 0;
737 const Bytef *cbuf = NULL;
740 if (ff_pkt->flags & FO_GZIP) {
741 if (ff_pkt->flags & FO_SPARSE) {
742 cbuf = (Bytef *)jcr->compress_buf + SPARSE_FADDR_SIZE;
743 max_compress_len = jcr->compress_buf_size - SPARSE_FADDR_SIZE;
745 cbuf = (Bytef *)jcr->compress_buf;
746 max_compress_len = jcr->compress_buf_size; /* set max length */
748 wbuf = jcr->compress_buf; /* compressed output here */
749 cipher_input = (uint8_t *)jcr->compress_buf; /* encrypt compressed data */
752 * Only change zlib parameters if there is no pending operation.
753 * This should never happen as deflatereset is called after each
757 if (((z_stream*)jcr->pZLIB_compress_workset)->total_in == 0) {
758 /* set gzip compression level - must be done per file */
759 if ((zstat=deflateParams((z_stream*)jcr->pZLIB_compress_workset,
760 ff_pkt->GZIP_level, Z_DEFAULT_STRATEGY)) != Z_OK) {
761 Jmsg(jcr, M_FATAL, 0, _("Compression deflateParams error: %d\n"), zstat);
762 set_jcr_job_status(jcr, JS_ErrorTerminated);
768 const uint32_t max_compress_len = 0;
771 if (ff_pkt->flags & FO_ENCRYPT) {
772 if (ff_pkt->flags & FO_SPARSE) {
773 Jmsg0(jcr, M_FATAL, 0, _("Encrypting sparse data not supported.\n"));
776 /* Allocate the cipher context */
777 if ((cipher_ctx = crypto_cipher_new(jcr->crypto.pki_session, true,
778 &cipher_block_size)) == NULL) {
779 /* Shouldn't happen! */
780 Jmsg0(jcr, M_FATAL, 0, _("Failed to initialize encryption context.\n"));
785 * Grow the crypto buffer, if necessary.
786 * crypto_cipher_update() will buffer up to (cipher_block_size - 1).
787 * We grow crypto_buf to the maximum number of blocks that
788 * could be returned for the given read buffer size.
789 * (Using the larger of either rsize or max_compress_len)
791 jcr->crypto.crypto_buf = check_pool_memory_size(jcr->crypto.crypto_buf,
792 (MAX(rsize + (int)sizeof(uint32_t), (int32_t)max_compress_len) +
793 cipher_block_size - 1) / cipher_block_size * cipher_block_size);
795 wbuf = jcr->crypto.crypto_buf; /* Encrypted, possibly compressed output here. */
799 * Send Data header to Storage daemon
800 * <file-index> <stream> <info>
802 if (!sd->fsend("%ld %d 0", jcr->JobFiles, stream)) {
803 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
807 Dmsg1(300, ">stored: datahdr %s\n", sd->msg);
810 * Make space at beginning of buffer for fileAddr because this
811 * same buffer will be used for writing if compression is off.
813 if (ff_pkt->flags & FO_SPARSE) {
814 rbuf += SPARSE_FADDR_SIZE;
815 rsize -= SPARSE_FADDR_SIZE;
816 #ifdef HAVE_FREEBSD_OS
818 * To read FreeBSD partitions, the read size must be
821 rsize = (rsize/512) * 512;
825 /* a RAW device read on win32 only works if the buffer is a multiple of 512 */
827 if (S_ISBLK(ff_pkt->statp.st_mode))
828 rsize = (rsize/512) * 512;
834 while ((sd->msglen=(uint32_t)bread(&ff_pkt->bfd, rbuf, rsize)) > 0) {
836 /* Check for sparse blocks */
837 if (ff_pkt->flags & FO_SPARSE) {
839 bool allZeros = false;
840 if ((sd->msglen == rsize &&
841 fileAddr+sd->msglen < (uint64_t)ff_pkt->statp.st_size) ||
842 ((ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO) &&
843 (uint64_t)ff_pkt->statp.st_size == 0)) {
844 allZeros = is_buf_zero(rbuf, rsize);
847 /* Put file address as first data in buffer */
848 ser_begin(wbuf, SPARSE_FADDR_SIZE);
849 ser_uint64(fileAddr); /* store fileAddr in begin of buffer */
851 fileAddr += sd->msglen; /* update file address */
852 /* Skip block of all zeros */
854 continue; /* skip block of zeros */
858 jcr->ReadBytes += sd->msglen; /* count bytes read */
860 /* Uncompressed cipher input length */
861 cipher_input_len = sd->msglen;
863 /* Update checksum if requested */
865 crypto_digest_update(digest, (uint8_t *)rbuf, sd->msglen);
868 /* Update signing digest if requested */
869 if (signing_digest) {
870 crypto_digest_update(signing_digest, (uint8_t *)rbuf, sd->msglen);
874 /* Do compression if turned on */
875 if (ff_pkt->flags & FO_GZIP && jcr->pZLIB_compress_workset) {
876 Dmsg3(400, "cbuf=0x%x rbuf=0x%x len=%u\n", cbuf, rbuf, sd->msglen);
878 ((z_stream*)jcr->pZLIB_compress_workset)->next_in = (Bytef *)rbuf;
879 ((z_stream*)jcr->pZLIB_compress_workset)->avail_in = sd->msglen;
880 ((z_stream*)jcr->pZLIB_compress_workset)->next_out = (Bytef *)cbuf;
881 ((z_stream*)jcr->pZLIB_compress_workset)->avail_out = max_compress_len;
883 if ((zstat=deflate((z_stream*)jcr->pZLIB_compress_workset, Z_FINISH)) != Z_STREAM_END) {
884 Jmsg(jcr, M_FATAL, 0, _("Compression deflate error: %d\n"), zstat);
885 set_jcr_job_status(jcr, JS_ErrorTerminated);
888 compress_len = ((z_stream*)jcr->pZLIB_compress_workset)->total_out;
889 /* reset zlib stream to be able to begin from scratch again */
890 if ((zstat=deflateReset((z_stream*)jcr->pZLIB_compress_workset)) != Z_OK) {
891 Jmsg(jcr, M_FATAL, 0, _("Compression deflateReset error: %d\n"), zstat);
892 set_jcr_job_status(jcr, JS_ErrorTerminated);
896 Dmsg2(400, "compressed len=%d uncompressed len=%d\n", compress_len,
899 sd->msglen = compress_len; /* set compressed length */
900 cipher_input_len = compress_len;
904 * Note, here we prepend the current record length to the beginning
905 * of the encrypted data. This is because both sparse and compression
906 * restore handling want records returned to them with exactly the
907 * same number of bytes that were processed in the backup handling.
908 * That is, both are block filters rather than a stream. When doing
909 * compression, the compression routines may buffer data, so that for
910 * any one record compressed, when it is decompressed the same size
911 * will not be obtained. Of course, the buffered data eventually comes
912 * out in subsequent crypto_cipher_update() calls or at least
913 * when crypto_cipher_finalize() is called. Unfortunately, this
914 * "feature" of encryption enormously complicates the restore code.
916 if (ff_pkt->flags & FO_ENCRYPT) {
917 uint32_t initial_len = 0;
920 if (ff_pkt->flags & FO_SPARSE) {
921 cipher_input_len += SPARSE_FADDR_SIZE;
924 /* Encrypt the length of the input block */
925 uint8_t packet_len[sizeof(uint32_t)];
927 ser_begin(packet_len, sizeof(uint32_t));
928 ser_uint32(cipher_input_len); /* store data len in begin of buffer */
929 Dmsg1(20, "Encrypt len=%d\n", cipher_input_len);
931 if (!crypto_cipher_update(cipher_ctx, packet_len, sizeof(packet_len),
932 (uint8_t *)jcr->crypto.crypto_buf, &initial_len)) {
933 /* Encryption failed. Shouldn't happen. */
934 Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
938 /* Encrypt the input block */
939 if (crypto_cipher_update(cipher_ctx, cipher_input, cipher_input_len,
940 (uint8_t *)&jcr->crypto.crypto_buf[initial_len], &encrypted_len)) {
941 if ((initial_len + encrypted_len) == 0) {
942 /* No full block of data available, read more data */
945 Dmsg2(400, "encrypted len=%d unencrypted len=%d\n", encrypted_len,
947 sd->msglen = initial_len + encrypted_len; /* set encrypted length */
949 /* Encryption failed. Shouldn't happen. */
950 Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
955 /* Send the buffer to the Storage daemon */
956 if (ff_pkt->flags & FO_SPARSE) {
957 sd->msglen += SPARSE_FADDR_SIZE; /* include fileAddr in size */
959 sd->msg = wbuf; /* set correct write buffer */
961 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
965 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
967 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
968 sd->msg = msgsave; /* restore read buffer */
970 } /* end while read file data */
972 if (sd->msglen < 0) { /* error */
974 Jmsg(jcr, M_ERROR, 0, _("Read error on file %s. ERR=%s\n"),
975 ff_pkt->fname, be.bstrerror(ff_pkt->bfd.berrno));
976 if (jcr->JobErrors++ > 1000) { /* insanity check */
977 Jmsg(jcr, M_FATAL, 0, _("Too many errors.\n"));
979 } else if (ff_pkt->flags & FO_ENCRYPT) {
981 * For encryption, we must call finalize to push out any
984 if (!crypto_cipher_finalize(cipher_ctx, (uint8_t *)jcr->crypto.crypto_buf,
986 /* Padding failed. Shouldn't happen. */
987 Jmsg(jcr, M_FATAL, 0, _("Encryption padding error\n"));
991 /* Note, on SSL pre-0.9.7, there is always some output */
992 if (encrypted_len > 0) {
993 sd->msglen = encrypted_len; /* set encrypted length */
994 sd->msg = jcr->crypto.crypto_buf; /* set correct write buffer */
996 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1000 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
1001 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
1002 sd->msg = msgsave; /* restore bnet buffer */
1006 if (!sd->signal(BNET_EOD)) { /* indicate end of file data */
1007 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1012 /* Free the cipher context */
1014 crypto_cipher_free(cipher_ctx);
1019 /* Free the cipher context */
1021 crypto_cipher_free(cipher_ctx);
1024 sd->msg = msgsave; /* restore bnet buffer */
1029 bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream)
1031 BSOCK *sd = jcr->store_bsock;
1032 char attribs[MAXSTRING];
1033 char attribsEx[MAXSTRING];
1036 #ifdef FD_NO_SEND_TEST
1040 Dmsg1(300, "encode_and_send_attrs fname=%s\n", ff_pkt->fname);
1041 /* Find what data stream we will use, then encode the attributes */
1042 if ((data_stream = select_data_stream(ff_pkt)) == STREAM_NONE) {
1043 /* This should not happen */
1044 Jmsg0(jcr, M_FATAL, 0, _("Invalid file flags, no supported data stream type.\n"));
1047 encode_stat(attribs, &ff_pkt->statp, ff_pkt->LinkFI, data_stream);
1049 /* Now possibly extend the attributes */
1050 attr_stream = encode_attribsEx(jcr, attribsEx, ff_pkt);
1052 Dmsg3(300, "File %s\nattribs=%s\nattribsEx=%s\n", ff_pkt->fname, attribs, attribsEx);
1055 jcr->JobFiles++; /* increment number of files sent */
1056 ff_pkt->FileIndex = jcr->JobFiles; /* return FileIndex */
1057 pm_strcpy(jcr->last_fname, ff_pkt->fname);
1061 * Send Attributes header to Storage daemon
1062 * <file-index> <stream> <info>
1064 if (!sd->fsend("%ld %d 0", jcr->JobFiles, attr_stream)) {
1065 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1069 Dmsg1(300, ">stored: attrhdr %s\n", sd->msg);
1072 * Send file attributes to Storage daemon
1075 * Filename (full path)
1076 * Encoded attributes
1077 * Link name (if type==FT_LNK or FT_LNKSAVED)
1078 * Encoded extended-attributes (for Win32)
1080 * For a directory, link is the same as fname, but with trailing
1081 * slash. For a linked file, link is the link.
1083 if (ff_pkt->type != FT_DELETED) { /* already stripped */
1086 if (ff_pkt->type == FT_LNK || ff_pkt->type == FT_LNKSAVED) {
1087 Dmsg2(300, "Link %s to %s\n", ff_pkt->fname, ff_pkt->link);
1088 stat = sd->fsend("%ld %d %s%c%s%c%s%c%s%c", jcr->JobFiles,
1089 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, ff_pkt->link, 0,
1091 } else if (ff_pkt->type == FT_DIREND || ff_pkt->type == FT_REPARSE) {
1092 /* Here link is the canonical filename (i.e. with trailing slash) */
1093 stat = sd->fsend("%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
1094 ff_pkt->type, ff_pkt->link, 0, attribs, 0, 0, attribsEx, 0);
1096 stat = sd->fsend("%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
1097 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, 0, attribsEx, 0);
1099 if (ff_pkt->type != FT_DELETED) {
1100 unstrip_path(ff_pkt);
1103 Dmsg2(300, ">stored: attr len=%d: %s\n", sd->msglen, sd->msg);
1105 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1109 sd->signal(BNET_EOD); /* indicate end of attributes data */
1114 * Do in place strip of path
1116 static bool do_strip(int count, char *in)
1122 /* Copy to first path separator -- Win32 might have c: ... */
1123 while (*in && !IsPathSeparator(*in)) {
1127 numsep++; /* one separator seen */
1128 for (stripped=0; stripped<count && *in; stripped++) {
1129 while (*in && !IsPathSeparator(*in)) {
1130 in++; /* skip chars */
1133 numsep++; /* count separators seen */
1134 in++; /* skip separator */
1138 while (*in) { /* copy to end */
1139 if (IsPathSeparator(*in)) {
1145 Dmsg4(500, "stripped=%d count=%d numsep=%d sep>count=%d\n",
1146 stripped, count, numsep, numsep>count);
1147 return stripped==count && numsep>count;
1151 * If requested strip leading components of the path so that we can
1152 * save file as if it came from a subdirectory. This is most useful
1153 * for dealing with snapshots, by removing the snapshot directory, or
1154 * in handling vendor migrations where files have been restored with
1155 * a vendor product into a subdirectory.
1157 void strip_path(FF_PKT *ff_pkt)
1159 if (!(ff_pkt->flags & FO_STRIPPATH) || ff_pkt->strip_path <= 0) {
1160 Dmsg1(200, "No strip for %s\n", ff_pkt->fname);
1163 if (!ff_pkt->fname_save) {
1164 ff_pkt->fname_save = get_pool_memory(PM_FNAME);
1165 ff_pkt->link_save = get_pool_memory(PM_FNAME);
1167 pm_strcpy(ff_pkt->fname_save, ff_pkt->fname);
1168 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1169 pm_strcpy(ff_pkt->link_save, ff_pkt->link);
1170 Dmsg2(500, "strcpy link_save=%d link=%d\n", strlen(ff_pkt->link_save),
1171 strlen(ff_pkt->link));
1172 sm_check(__FILE__, __LINE__, true);
1176 * Strip path. If it doesn't succeed put it back. If
1177 * it does, and there is a different link string,
1178 * attempt to strip the link. If it fails, back them
1180 * Do not strip symlinks.
1181 * I.e. if either stripping fails don't strip anything.
1183 if (!do_strip(ff_pkt->strip_path, ff_pkt->fname)) {
1184 unstrip_path(ff_pkt);
1187 /* Strip links but not symlinks */
1188 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1189 if (!do_strip(ff_pkt->strip_path, ff_pkt->link)) {
1190 unstrip_path(ff_pkt);
1195 Dmsg3(100, "fname=%s stripped=%s link=%s\n", ff_pkt->fname_save, ff_pkt->fname,
1199 void unstrip_path(FF_PKT *ff_pkt)
1201 if (!(ff_pkt->flags & FO_STRIPPATH) || ff_pkt->strip_path <= 0) {
1204 strcpy(ff_pkt->fname, ff_pkt->fname_save);
1205 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1206 Dmsg2(500, "strcpy link=%s link_save=%s\n", ff_pkt->link,
1208 strcpy(ff_pkt->link, ff_pkt->link_save);
1209 Dmsg2(500, "strcpy link=%d link_save=%d\n", strlen(ff_pkt->link),
1210 strlen(ff_pkt->link_save));
1211 sm_check(__FILE__, __LINE__, true);