2 Bacula® - The Network Backup Solution
4 Copyright (C) 2000-2008 Free Software Foundation Europe e.V.
6 The main author of Bacula is Kern Sibbald, with contributions from
7 many others, a complete list can be found in the file AUTHORS.
8 This program is Free Software; you can redistribute it and/or
9 modify it under the terms of version two of the GNU General Public
10 License as published by the Free Software Foundation and included
13 This program is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
23 Bacula® is a registered trademark of Kern Sibbald.
24 The licensor of Bacula is the Free Software Foundation Europe
25 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
26 Switzerland, email:ftf@fsfeurope.org.
29 * Bacula File Daemon backup.c send file attributes and data
30 * to the Storage daemon.
32 * Kern Sibbald, March MM
41 /* Forward referenced functions */
42 int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level);
43 static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, DIGEST *signature_digest);
44 bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream);
45 static bool crypto_session_start(JCR *jcr);
46 static void crypto_session_end(JCR *jcr);
47 static bool crypto_session_send(JCR *jcr, BSOCK *sd);
50 * check for BSD nodump flag
52 static bool no_dump(JCR *jcr, FF_PKT *ff_pkt)
54 #if defined(HAVE_CHFLAGS) && defined(UF_NODUMP)
55 if ( (ff_pkt->flags & FO_HONOR_NODUMP) &&
56 (ff_pkt->statp.st_flags & UF_NODUMP) ) {
57 Jmsg(jcr, M_INFO, 1, _(" NODUMP flag set - will not process %s\n"),
59 return true; /* do not backup this file */
62 return false; /* do backup */
66 * Find all the requested files and send them
67 * to the Storage daemon.
69 * Note, we normally carry on a one-way
70 * conversation from this point on with the SD, simply blasting
71 * data to him. To properly know what is going on, we
72 * also run a "heartbeat" monitor which reads the socket and
73 * reacts accordingly (at the moment it has nothing to do
74 * except echo the heartbeat to the Director).
77 bool blast_data_to_storage_daemon(JCR *jcr, char *addr)
81 // TODO landonf: Allow user to specify encryption algorithm
83 sd = jcr->store_bsock;
85 set_jcr_job_status(jcr, JS_Running);
87 Dmsg1(300, "bfiled: opened data connection %d to stored\n", sd->m_fd);
90 CLIENT *client = (CLIENT *)GetNextRes(R_CLIENT, NULL);
94 buf_size = client->max_network_buffer_size;
96 buf_size = 0; /* use default */
98 if (!sd->set_buffer_size(buf_size, BNET_SETBUF_WRITE)) {
99 set_jcr_job_status(jcr, JS_ErrorTerminated);
100 Jmsg(jcr, M_FATAL, 0, _("Cannot set buffer size FD->SD.\n"));
104 jcr->buf_size = sd->msglen;
105 /* Adjust for compression so that output buffer is
106 * 12 bytes + 0.1% larger than input buffer plus 18 bytes.
107 * This gives a bit extra plus room for the sparse addr if any.
108 * Note, we adjust the read size to be smaller so that the
109 * same output buffer can be used without growing it.
111 * The zlib compression workset is initialized here to minimize
112 * the "per file" load. The jcr member is only set, if the init
115 jcr->compress_buf_size = jcr->buf_size + ((jcr->buf_size+999) / 1000) + 30;
116 jcr->compress_buf = get_memory(jcr->compress_buf_size);
119 z_stream *pZlibStream = (z_stream*)malloc(sizeof(z_stream));
121 pZlibStream->zalloc = Z_NULL;
122 pZlibStream->zfree = Z_NULL;
123 pZlibStream->opaque = Z_NULL;
124 pZlibStream->state = Z_NULL;
126 if (deflateInit(pZlibStream, Z_DEFAULT_COMPRESSION) == Z_OK) {
127 jcr->pZLIB_compress_workset = pZlibStream;
134 if (!crypto_session_start(jcr)) {
138 set_find_options((FF_PKT *)jcr->ff, jcr->incremental, jcr->mtime);
140 /* in accurate mode, we overwrite the find_one check function */
142 set_find_changed_function((FF_PKT *)jcr->ff, accurate_check_file);
145 start_heartbeat_monitor(jcr);
147 jcr->acl_data = get_pool_memory(PM_MESSAGE);
148 jcr->xattr_data = get_pool_memory(PM_MESSAGE);
150 /* Subroutine save_file() is called for each file */
151 if (!find_files(jcr, (FF_PKT *)jcr->ff, save_file, plugin_save)) {
152 ok = false; /* error */
153 set_jcr_job_status(jcr, JS_ErrorTerminated);
156 accurate_send_deleted_list(jcr); /* send deleted list to SD */
158 free_pool_memory(jcr->acl_data);
159 free_pool_memory(jcr->xattr_data);
161 stop_heartbeat_monitor(jcr);
163 sd->signal(BNET_EOD); /* end of sending data */
169 if (jcr->compress_buf) {
170 free_pool_memory(jcr->compress_buf);
171 jcr->compress_buf = NULL;
173 if (jcr->pZLIB_compress_workset) {
174 /* Free the zlib stream */
176 deflateEnd((z_stream *)jcr->pZLIB_compress_workset);
178 free (jcr->pZLIB_compress_workset);
179 jcr->pZLIB_compress_workset = NULL;
181 crypto_session_end(jcr);
184 Dmsg1(100, "end blast_data ok=%d\n", ok);
188 static bool crypto_session_start(JCR *jcr)
190 crypto_cipher_t cipher = CRYPTO_CIPHER_AES_128_CBC;
193 * Create encryption session data and a cached, DER-encoded session data
194 * structure. We use a single session key for each backup, so we'll encode
195 * the session data only once.
197 if (jcr->crypto.pki_encrypt) {
200 /* Create per-job session encryption context */
201 jcr->crypto.pki_session = crypto_session_new(cipher, jcr->crypto.pki_recipients);
203 /* Get the session data size */
204 if (!crypto_session_encode(jcr->crypto.pki_session, (uint8_t *)0, &size)) {
205 Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n"));
209 /* Allocate buffer */
210 jcr->crypto.pki_session_encoded = get_memory(size);
212 /* Encode session data */
213 if (!crypto_session_encode(jcr->crypto.pki_session, (uint8_t *)jcr->crypto.pki_session_encoded, &size)) {
214 Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n"));
218 /* ... and store the encoded size */
219 jcr->crypto.pki_session_encoded_size = size;
221 /* Allocate the encryption/decryption buffer */
222 jcr->crypto.crypto_buf = get_memory(CRYPTO_CIPHER_MAX_BLOCK_SIZE);
227 static void crypto_session_end(JCR *jcr)
229 if (jcr->crypto.crypto_buf) {
230 free_pool_memory(jcr->crypto.crypto_buf);
231 jcr->crypto.crypto_buf = NULL;
233 if (jcr->crypto.pki_session) {
234 crypto_session_free(jcr->crypto.pki_session);
236 if (jcr->crypto.pki_session_encoded) {
237 free_pool_memory(jcr->crypto.pki_session_encoded);
238 jcr->crypto.pki_session_encoded = NULL;
242 static bool crypto_session_send(JCR *jcr, BSOCK *sd)
246 /* Send our header */
247 Dmsg2(100, "Send hdr fi=%ld stream=%d\n", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA);
248 sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA);
251 sd->msg = jcr->crypto.pki_session_encoded;
252 sd->msglen = jcr->crypto.pki_session_encoded_size;
253 jcr->JobBytes += sd->msglen;
255 Dmsg1(100, "Send data len=%d\n", sd->msglen);
258 sd->signal(BNET_EOD);
264 * Called here by find() for each file included.
265 * This is a callback. The original is find_files() above.
267 * Send the file and its data to the Storage daemon.
271 * -1 to ignore file/directory (not used here)
273 int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level)
275 bool do_read = false;
276 int stat, data_stream;
278 DIGEST *digest = NULL;
279 DIGEST *signing_digest = NULL;
280 int digest_stream = STREAM_NONE;
281 SIGNATURE *sig = NULL;
282 bool has_file_data = false;
283 // TODO landonf: Allow the user to specify the digest algorithm
285 crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA256;
287 crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA1;
289 BSOCK *sd = jcr->store_bsock;
291 if (job_canceled(jcr)) {
295 jcr->num_files_examined++; /* bump total file count */
297 switch (ff_pkt->type) {
298 case FT_LNKSAVED: /* Hard linked, file already saved */
299 Dmsg2(130, "FT_LNKSAVED hard link: %s => %s\n", ff_pkt->fname, ff_pkt->link);
302 Dmsg1(130, "FT_REGE saving: %s\n", ff_pkt->fname);
303 if (no_dump(jcr, ff_pkt))
305 has_file_data = true;
308 Dmsg1(130, "FT_REG saving: %s\n", ff_pkt->fname);
309 if (no_dump(jcr, ff_pkt))
311 has_file_data = true;
314 Dmsg2(130, "FT_LNK saving: %s -> %s\n", ff_pkt->fname, ff_pkt->link);
317 jcr->num_files_examined--; /* correct file count */
318 if (no_dump(jcr, ff_pkt)) /* disable recursion on nodump directories */
319 ff_pkt->flags |= FO_NO_RECURSION;
320 return 1; /* not used */
322 Jmsg(jcr, M_INFO, 1, _(" Recursion turned off. Will not descend from %s into %s\n"),
323 ff_pkt->top_fname, ff_pkt->fname);
324 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
327 /* Suppress message for /dev filesystems */
328 if (!is_in_fileset(ff_pkt)) {
329 Jmsg(jcr, M_INFO, 1, _(" %s is a different filesystem. Will not descend from %s into %s\n"),
330 ff_pkt->fname, ff_pkt->top_fname, ff_pkt->fname);
332 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
335 Jmsg(jcr, M_INFO, 1, _(" Disallowed filesystem. Will not descend from %s into %s\n"),
336 ff_pkt->top_fname, ff_pkt->fname);
337 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
340 Jmsg(jcr, M_INFO, 1, _(" Disallowed drive type. Will not descend into %s\n"),
345 Dmsg1(130, "FT_DIREND: %s\n", ff_pkt->link);
348 Dmsg1(130, "FT_SPEC saving: %s\n", ff_pkt->fname);
349 if (S_ISSOCK(ff_pkt->statp.st_mode)) {
350 Jmsg(jcr, M_SKIPPED, 1, _(" Socket file skipped: %s\n"), ff_pkt->fname);
355 Dmsg1(130, "FT_RAW saving: %s\n", ff_pkt->fname);
356 has_file_data = true;
359 Dmsg1(130, "FT_FIFO saving: %s\n", ff_pkt->fname);
363 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not access \"%s\": ERR=%s\n"), ff_pkt->fname,
364 be.bstrerror(ff_pkt->ff_errno));
370 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not follow link \"%s\": ERR=%s\n"),
371 ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
377 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not stat \"%s\": ERR=%s\n"), ff_pkt->fname,
378 be.bstrerror(ff_pkt->ff_errno));
384 Jmsg(jcr, M_SKIPPED, 1, _(" Unchanged file skipped: %s\n"), ff_pkt->fname);
387 Jmsg(jcr, M_NOTSAVED, 0, _(" Archive file not saved: %s\n"), ff_pkt->fname);
391 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not open directory \"%s\": ERR=%s\n"),
392 ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
397 Jmsg(jcr, M_NOTSAVED, 0, _(" Unknown file type %d; not saved: %s\n"),
398 ff_pkt->type, ff_pkt->fname);
403 Dmsg1(130, "bfiled: sending %s to stored\n", ff_pkt->fname);
405 /* Digests and encryption are only useful if there's file data */
408 * Setup for digest handling. If this fails, the digest will be set to NULL
409 * and not used. Note, the digest (file hash) can be any one of the four
412 * The signing digest is a single algorithm depending on
413 * whether or not we have SHA2.
414 * ****FIXME**** the signing algoritm should really be
415 * determined a different way!!!!!! What happens if
416 * sha2 was available during backup but not restore?
418 if (ff_pkt->flags & FO_MD5) {
419 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_MD5);
420 digest_stream = STREAM_MD5_DIGEST;
422 } else if (ff_pkt->flags & FO_SHA1) {
423 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA1);
424 digest_stream = STREAM_SHA1_DIGEST;
426 } else if (ff_pkt->flags & FO_SHA256) {
427 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA256);
428 digest_stream = STREAM_SHA256_DIGEST;
430 } else if (ff_pkt->flags & FO_SHA512) {
431 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA512);
432 digest_stream = STREAM_SHA512_DIGEST;
435 /* Did digest initialization fail? */
436 if (digest_stream != STREAM_NONE && digest == NULL) {
437 Jmsg(jcr, M_WARNING, 0, _("%s digest initialization failed\n"),
438 stream_to_ascii(digest_stream));
442 * Set up signature digest handling. If this fails, the signature digest will be set to
445 // TODO landonf: We should really only calculate the digest once, for both verification and signing.
446 if (jcr->crypto.pki_sign) {
447 signing_digest = crypto_digest_new(jcr, signing_algorithm);
449 /* Full-stop if a failure occurred initializing the signature digest */
450 if (signing_digest == NULL) {
451 Jmsg(jcr, M_NOTSAVED, 0, _("%s signature digest initialization failed\n"),
452 stream_to_ascii(signing_algorithm));
458 /* Enable encryption */
459 if (jcr->crypto.pki_encrypt) {
460 ff_pkt->flags |= FO_ENCRYPT;
464 /* Initialize the file descriptor we use for data and other streams. */
466 if (ff_pkt->flags & FO_PORTABLE) {
467 set_portable_backup(&ff_pkt->bfd); /* disable Win32 BackupRead() */
469 if (ff_pkt->cmd_plugin) {
470 if (!set_cmd_plugin(&ff_pkt->bfd, jcr)) {
473 send_plugin_name(jcr, sd, true); /* signal start of plugin data */
476 /* Send attributes -- must be done after binit() */
477 if (!encode_and_send_attributes(jcr, ff_pkt, data_stream)) {
481 /* Set up the encryption context and send the session data to the SD */
482 if (has_file_data && jcr->crypto.pki_encrypt) {
483 if (!crypto_session_send(jcr, sd)) {
489 * Open any file with data that we intend to save, then save it.
491 * Note, if is_win32_backup, we must open the Directory so that
492 * the BackupRead will save its permissions and ownership streams.
494 if (ff_pkt->type != FT_LNKSAVED && S_ISREG(ff_pkt->statp.st_mode)) {
496 do_read = !is_portable_backup(&ff_pkt->bfd) || ff_pkt->statp.st_size > 0;
498 do_read = ff_pkt->statp.st_size > 0;
500 } else if (ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO ||
501 ff_pkt->type == FT_REPARSE ||
502 (!is_portable_backup(&ff_pkt->bfd) && ff_pkt->type == FT_DIREND)) {
505 if (ff_pkt->cmd_plugin) {
509 Dmsg1(400, "do_read=%d\n", do_read);
513 if (ff_pkt->type == FT_FIFO) {
514 tid = start_thread_timer(jcr, pthread_self(), 60);
518 int noatime = ff_pkt->flags & FO_NOATIME ? O_NOATIME : 0;
519 ff_pkt->bfd.reparse_point = ff_pkt->type == FT_REPARSE;
520 if (bopen(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY | noatime, 0) < 0) {
521 ff_pkt->ff_errno = errno;
523 Jmsg(jcr, M_NOTSAVED, 0, _(" Cannot open \"%s\": ERR=%s.\n"), ff_pkt->fname,
527 stop_thread_timer(tid);
533 stop_thread_timer(tid);
537 stat = send_data(jcr, data_stream, ff_pkt, digest, signing_digest);
539 if (ff_pkt->flags & FO_CHKCHANGES) {
540 has_file_changed(jcr, ff_pkt);
543 bclose(&ff_pkt->bfd);
550 #ifdef HAVE_DARWIN_OS
551 /* Regular files can have resource forks and Finder Info */
552 if (ff_pkt->type != FT_LNKSAVED && (S_ISREG(ff_pkt->statp.st_mode) &&
553 ff_pkt->flags & FO_HFSPLUS)) {
554 if (ff_pkt->hfsinfo.rsrclength > 0) {
557 if (!bopen_rsrc(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0) < 0) {
558 ff_pkt->ff_errno = errno;
560 Jmsg(jcr, M_NOTSAVED, -1, _(" Cannot open resource fork for \"%s\": ERR=%s.\n"),
561 ff_pkt->fname, be.bstrerror());
563 if (is_bopen(&ff_pkt->bfd)) {
564 bclose(&ff_pkt->bfd);
568 flags = ff_pkt->flags;
569 ff_pkt->flags &= ~(FO_GZIP|FO_SPARSE);
570 if (flags & FO_ENCRYPT) {
571 rsrc_stream = STREAM_ENCRYPTED_MACOS_FORK_DATA;
573 rsrc_stream = STREAM_MACOS_FORK_DATA;
575 stat = send_data(jcr, rsrc_stream, ff_pkt, digest, signing_digest);
576 ff_pkt->flags = flags;
577 bclose(&ff_pkt->bfd);
583 Dmsg1(300, "Saving Finder Info for \"%s\"\n", ff_pkt->fname);
584 sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_HFSPLUS_ATTRIBUTES);
585 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
586 pm_memcpy(sd->msg, ff_pkt->hfsinfo.fndrinfo, 32);
589 crypto_digest_update(digest, (uint8_t *)sd->msg, sd->msglen);
591 if (signing_digest) {
592 crypto_digest_update(signing_digest, (uint8_t *)sd->msg, sd->msglen);
595 sd->signal(BNET_EOD);
600 * Save ACLs for anything not being a symlink.
602 if (ff_pkt->flags & FO_ACL && ff_pkt->type != FT_LNK) {
603 if (!build_acl_streams(jcr, ff_pkt))
608 * Save Extended Attributes for all files.
610 if (ff_pkt->flags & FO_XATTR) {
611 if (!build_xattr_streams(jcr, ff_pkt))
615 /* Terminate the signing digest and send it to the Storage daemon */
616 if (signing_digest) {
619 if ((sig = crypto_sign_new(jcr)) == NULL) {
620 Jmsg(jcr, M_FATAL, 0, _("Failed to allocate memory for crypto signature.\n"));
624 if (!crypto_sign_add_signer(sig, signing_digest, jcr->crypto.pki_keypair)) {
625 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
629 /* Get signature size */
630 if (!crypto_sign_encode(sig, NULL, &size)) {
631 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
635 /* Grow the bsock buffer to fit our message if necessary */
636 if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
637 sd->msg = realloc_pool_memory(sd->msg, size);
640 /* Send our header */
641 sd->fsend("%ld %ld 0", jcr->JobFiles, STREAM_SIGNED_DIGEST);
642 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
644 /* Encode signature data */
645 if (!crypto_sign_encode(sig, (uint8_t *)sd->msg, &size)) {
646 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
652 sd->signal(BNET_EOD); /* end of checksum */
655 /* Terminate any digest and send it to Storage daemon */
659 sd->fsend("%ld %d 0", jcr->JobFiles, digest_stream);
660 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
662 size = CRYPTO_DIGEST_MAX_SIZE;
664 /* Grow the bsock buffer to fit our message if necessary */
665 if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
666 sd->msg = realloc_pool_memory(sd->msg, size);
669 if (!crypto_digest_finalize(digest, (uint8_t *)sd->msg, &size)) {
670 Jmsg(jcr, M_FATAL, 0, _("An error occurred finalizing signing the stream.\n"));
676 sd->signal(BNET_EOD); /* end of checksum */
678 if (ff_pkt->cmd_plugin) {
679 send_plugin_name(jcr, sd, false); /* signal end of plugin data */
683 rtnstat = 1; /* good return */
687 crypto_digest_free(digest);
689 if (signing_digest) {
690 crypto_digest_free(signing_digest);
693 crypto_sign_free(sig);
699 * Send data read from an already open file descriptor.
701 * We return 1 on sucess and 0 on errors.
704 * We use ff_pkt->statp.st_size when FO_SPARSE to know when to stop
706 * Currently this is not a problem as the only other stream, resource forks,
707 * are not handled as sparse files.
709 int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest,
710 DIGEST *signing_digest)
712 BSOCK *sd = jcr->store_bsock;
713 uint64_t fileAddr = 0; /* file address */
715 int32_t rsize = jcr->buf_size; /* read buffer size */
717 CIPHER_CONTEXT *cipher_ctx = NULL; /* Quell bogus uninitialized warnings */
718 const uint8_t *cipher_input;
719 uint32_t cipher_input_len;
720 uint32_t cipher_block_size;
721 uint32_t encrypted_len;
722 #ifdef FD_NO_SEND_TEST
727 rbuf = sd->msg; /* read buffer */
728 wbuf = sd->msg; /* write buffer */
729 cipher_input = (uint8_t *)rbuf; /* encrypt uncompressed data */
731 Dmsg1(300, "Saving data, type=%d\n", ff_pkt->type);
734 uLong compress_len = 0;
735 uLong max_compress_len = 0;
736 const Bytef *cbuf = NULL;
739 if (ff_pkt->flags & FO_GZIP) {
740 if (ff_pkt->flags & FO_SPARSE) {
741 cbuf = (Bytef *)jcr->compress_buf + SPARSE_FADDR_SIZE;
742 max_compress_len = jcr->compress_buf_size - SPARSE_FADDR_SIZE;
744 cbuf = (Bytef *)jcr->compress_buf;
745 max_compress_len = jcr->compress_buf_size; /* set max length */
747 wbuf = jcr->compress_buf; /* compressed output here */
748 cipher_input = (uint8_t *)jcr->compress_buf; /* encrypt compressed data */
751 * Only change zlib parameters if there is no pending operation.
752 * This should never happen as deflatereset is called after each
756 if (((z_stream*)jcr->pZLIB_compress_workset)->total_in == 0) {
757 /* set gzip compression level - must be done per file */
758 if ((zstat=deflateParams((z_stream*)jcr->pZLIB_compress_workset,
759 ff_pkt->GZIP_level, Z_DEFAULT_STRATEGY)) != Z_OK) {
760 Jmsg(jcr, M_FATAL, 0, _("Compression deflateParams error: %d\n"), zstat);
761 set_jcr_job_status(jcr, JS_ErrorTerminated);
767 const uint32_t max_compress_len = 0;
770 if (ff_pkt->flags & FO_ENCRYPT) {
771 if (ff_pkt->flags & FO_SPARSE) {
772 Jmsg0(jcr, M_FATAL, 0, _("Encrypting sparse data not supported.\n"));
775 /* Allocate the cipher context */
776 if ((cipher_ctx = crypto_cipher_new(jcr->crypto.pki_session, true,
777 &cipher_block_size)) == NULL) {
778 /* Shouldn't happen! */
779 Jmsg0(jcr, M_FATAL, 0, _("Failed to initialize encryption context.\n"));
784 * Grow the crypto buffer, if necessary.
785 * crypto_cipher_update() will buffer up to (cipher_block_size - 1).
786 * We grow crypto_buf to the maximum number of blocks that
787 * could be returned for the given read buffer size.
788 * (Using the larger of either rsize or max_compress_len)
790 jcr->crypto.crypto_buf = check_pool_memory_size(jcr->crypto.crypto_buf,
791 (MAX(rsize + (int)sizeof(uint32_t), (int32_t)max_compress_len) +
792 cipher_block_size - 1) / cipher_block_size * cipher_block_size);
794 wbuf = jcr->crypto.crypto_buf; /* Encrypted, possibly compressed output here. */
798 * Send Data header to Storage daemon
799 * <file-index> <stream> <info>
801 if (!sd->fsend("%ld %d 0", jcr->JobFiles, stream)) {
802 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
806 Dmsg1(300, ">stored: datahdr %s\n", sd->msg);
809 * Make space at beginning of buffer for fileAddr because this
810 * same buffer will be used for writing if compression is off.
812 if (ff_pkt->flags & FO_SPARSE) {
813 rbuf += SPARSE_FADDR_SIZE;
814 rsize -= SPARSE_FADDR_SIZE;
815 #ifdef HAVE_FREEBSD_OS
817 * To read FreeBSD partitions, the read size must be
820 rsize = (rsize/512) * 512;
824 /* a RAW device read on win32 only works if the buffer is a multiple of 512 */
826 if (S_ISBLK(ff_pkt->statp.st_mode))
827 rsize = (rsize/512) * 512;
833 while ((sd->msglen=(uint32_t)bread(&ff_pkt->bfd, rbuf, rsize)) > 0) {
835 /* Check for sparse blocks */
836 if (ff_pkt->flags & FO_SPARSE) {
838 bool allZeros = false;
839 if ((sd->msglen == rsize &&
840 fileAddr+sd->msglen < (uint64_t)ff_pkt->statp.st_size) ||
841 ((ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO) &&
842 (uint64_t)ff_pkt->statp.st_size == 0)) {
843 allZeros = is_buf_zero(rbuf, rsize);
846 /* Put file address as first data in buffer */
847 ser_begin(wbuf, SPARSE_FADDR_SIZE);
848 ser_uint64(fileAddr); /* store fileAddr in begin of buffer */
850 fileAddr += sd->msglen; /* update file address */
851 /* Skip block of all zeros */
853 continue; /* skip block of zeros */
857 jcr->ReadBytes += sd->msglen; /* count bytes read */
859 /* Uncompressed cipher input length */
860 cipher_input_len = sd->msglen;
862 /* Update checksum if requested */
864 crypto_digest_update(digest, (uint8_t *)rbuf, sd->msglen);
867 /* Update signing digest if requested */
868 if (signing_digest) {
869 crypto_digest_update(signing_digest, (uint8_t *)rbuf, sd->msglen);
873 /* Do compression if turned on */
874 if (ff_pkt->flags & FO_GZIP && jcr->pZLIB_compress_workset) {
875 Dmsg3(400, "cbuf=0x%x rbuf=0x%x len=%u\n", cbuf, rbuf, sd->msglen);
877 ((z_stream*)jcr->pZLIB_compress_workset)->next_in = (Bytef *)rbuf;
878 ((z_stream*)jcr->pZLIB_compress_workset)->avail_in = sd->msglen;
879 ((z_stream*)jcr->pZLIB_compress_workset)->next_out = (Bytef *)cbuf;
880 ((z_stream*)jcr->pZLIB_compress_workset)->avail_out = max_compress_len;
882 if ((zstat=deflate((z_stream*)jcr->pZLIB_compress_workset, Z_FINISH)) != Z_STREAM_END) {
883 Jmsg(jcr, M_FATAL, 0, _("Compression deflate error: %d\n"), zstat);
884 set_jcr_job_status(jcr, JS_ErrorTerminated);
887 compress_len = ((z_stream*)jcr->pZLIB_compress_workset)->total_out;
888 /* reset zlib stream to be able to begin from scratch again */
889 if ((zstat=deflateReset((z_stream*)jcr->pZLIB_compress_workset)) != Z_OK) {
890 Jmsg(jcr, M_FATAL, 0, _("Compression deflateReset error: %d\n"), zstat);
891 set_jcr_job_status(jcr, JS_ErrorTerminated);
895 Dmsg2(400, "compressed len=%d uncompressed len=%d\n", compress_len,
898 sd->msglen = compress_len; /* set compressed length */
899 cipher_input_len = compress_len;
903 * Note, here we prepend the current record length to the beginning
904 * of the encrypted data. This is because both sparse and compression
905 * restore handling want records returned to them with exactly the
906 * same number of bytes that were processed in the backup handling.
907 * That is, both are block filters rather than a stream. When doing
908 * compression, the compression routines may buffer data, so that for
909 * any one record compressed, when it is decompressed the same size
910 * will not be obtained. Of course, the buffered data eventually comes
911 * out in subsequent crypto_cipher_update() calls or at least
912 * when crypto_cipher_finalize() is called. Unfortunately, this
913 * "feature" of encryption enormously complicates the restore code.
915 if (ff_pkt->flags & FO_ENCRYPT) {
916 uint32_t initial_len = 0;
919 if (ff_pkt->flags & FO_SPARSE) {
920 cipher_input_len += SPARSE_FADDR_SIZE;
923 /* Encrypt the length of the input block */
924 uint8_t packet_len[sizeof(uint32_t)];
926 ser_begin(packet_len, sizeof(uint32_t));
927 ser_uint32(cipher_input_len); /* store data len in begin of buffer */
928 Dmsg1(20, "Encrypt len=%d\n", cipher_input_len);
930 if (!crypto_cipher_update(cipher_ctx, packet_len, sizeof(packet_len),
931 (uint8_t *)jcr->crypto.crypto_buf, &initial_len)) {
932 /* Encryption failed. Shouldn't happen. */
933 Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
937 /* Encrypt the input block */
938 if (crypto_cipher_update(cipher_ctx, cipher_input, cipher_input_len,
939 (uint8_t *)&jcr->crypto.crypto_buf[initial_len], &encrypted_len)) {
940 if ((initial_len + encrypted_len) == 0) {
941 /* No full block of data available, read more data */
944 Dmsg2(400, "encrypted len=%d unencrypted len=%d\n", encrypted_len,
946 sd->msglen = initial_len + encrypted_len; /* set encrypted length */
948 /* Encryption failed. Shouldn't happen. */
949 Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
954 /* Send the buffer to the Storage daemon */
955 if (ff_pkt->flags & FO_SPARSE) {
956 sd->msglen += SPARSE_FADDR_SIZE; /* include fileAddr in size */
958 sd->msg = wbuf; /* set correct write buffer */
960 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
964 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
966 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
967 sd->msg = msgsave; /* restore read buffer */
969 } /* end while read file data */
971 if (sd->msglen < 0) { /* error */
973 Jmsg(jcr, M_ERROR, 0, _("Read error on file %s. ERR=%s\n"),
974 ff_pkt->fname, be.bstrerror(ff_pkt->bfd.berrno));
975 if (jcr->Errors++ > 1000) { /* insanity check */
976 Jmsg(jcr, M_FATAL, 0, _("Too many errors.\n"));
978 } else if (ff_pkt->flags & FO_ENCRYPT) {
980 * For encryption, we must call finalize to push out any
983 if (!crypto_cipher_finalize(cipher_ctx, (uint8_t *)jcr->crypto.crypto_buf,
985 /* Padding failed. Shouldn't happen. */
986 Jmsg(jcr, M_FATAL, 0, _("Encryption padding error\n"));
990 /* Note, on SSL pre-0.9.7, there is always some output */
991 if (encrypted_len > 0) {
992 sd->msglen = encrypted_len; /* set encrypted length */
993 sd->msg = jcr->crypto.crypto_buf; /* set correct write buffer */
995 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
999 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
1000 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
1001 sd->msg = msgsave; /* restore bnet buffer */
1005 if (!sd->signal(BNET_EOD)) { /* indicate end of file data */
1006 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1011 /* Free the cipher context */
1013 crypto_cipher_free(cipher_ctx);
1018 /* Free the cipher context */
1020 crypto_cipher_free(cipher_ctx);
1023 sd->msg = msgsave; /* restore bnet buffer */
1028 bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream)
1030 BSOCK *sd = jcr->store_bsock;
1031 char attribs[MAXSTRING];
1032 char attribsEx[MAXSTRING];
1035 #ifdef FD_NO_SEND_TEST
1039 Dmsg1(300, "encode_and_send_attrs fname=%s\n", ff_pkt->fname);
1040 /* Find what data stream we will use, then encode the attributes */
1041 if ((data_stream = select_data_stream(ff_pkt)) == STREAM_NONE) {
1042 /* This should not happen */
1043 Jmsg0(jcr, M_FATAL, 0, _("Invalid file flags, no supported data stream type.\n"));
1046 encode_stat(attribs, ff_pkt, data_stream);
1048 /* Now possibly extend the attributes */
1049 attr_stream = encode_attribsEx(jcr, attribsEx, ff_pkt);
1051 Dmsg3(300, "File %s\nattribs=%s\nattribsEx=%s\n", ff_pkt->fname, attribs, attribsEx);
1054 jcr->JobFiles++; /* increment number of files sent */
1055 ff_pkt->FileIndex = jcr->JobFiles; /* return FileIndex */
1056 pm_strcpy(jcr->last_fname, ff_pkt->fname);
1060 * Send Attributes header to Storage daemon
1061 * <file-index> <stream> <info>
1063 if (!sd->fsend("%ld %d 0", jcr->JobFiles, attr_stream)) {
1064 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1068 Dmsg1(300, ">stored: attrhdr %s\n", sd->msg);
1071 * Send file attributes to Storage daemon
1074 * Filename (full path)
1075 * Encoded attributes
1076 * Link name (if type==FT_LNK or FT_LNKSAVED)
1077 * Encoded extended-attributes (for Win32)
1079 * For a directory, link is the same as fname, but with trailing
1080 * slash. For a linked file, link is the link.
1082 if (ff_pkt->type != FT_DELETED) { /* already stripped */
1085 if (ff_pkt->type == FT_LNK || ff_pkt->type == FT_LNKSAVED) {
1086 Dmsg2(300, "Link %s to %s\n", ff_pkt->fname, ff_pkt->link);
1087 stat = sd->fsend("%ld %d %s%c%s%c%s%c%s%c", jcr->JobFiles,
1088 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, ff_pkt->link, 0,
1090 } else if (ff_pkt->type == FT_DIREND || ff_pkt->type == FT_REPARSE) {
1091 /* Here link is the canonical filename (i.e. with trailing slash) */
1092 stat = sd->fsend("%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
1093 ff_pkt->type, ff_pkt->link, 0, attribs, 0, 0, attribsEx, 0);
1095 stat = sd->fsend("%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
1096 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, 0, attribsEx, 0);
1098 if (ff_pkt->type != FT_DELETED) {
1099 unstrip_path(ff_pkt);
1102 Dmsg2(300, ">stored: attr len=%d: %s\n", sd->msglen, sd->msg);
1104 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1108 sd->signal(BNET_EOD); /* indicate end of attributes data */
1113 * Do in place strip of path
1115 static bool do_strip(int count, char *in)
1121 /* Copy to first path separator -- Win32 might have c: ... */
1122 while (*in && !IsPathSeparator(*in)) {
1126 numsep++; /* one separator seen */
1127 for (stripped=0; stripped<count && *in; stripped++) {
1128 while (*in && !IsPathSeparator(*in)) {
1129 in++; /* skip chars */
1132 numsep++; /* count separators seen */
1133 in++; /* skip separator */
1137 while (*in) { /* copy to end */
1138 if (IsPathSeparator(*in)) {
1144 Dmsg4(500, "stripped=%d count=%d numsep=%d sep>count=%d\n",
1145 stripped, count, numsep, numsep>count);
1146 return stripped==count && numsep>count;
1150 * If requested strip leading components of the path so that we can
1151 * save file as if it came from a subdirectory. This is most useful
1152 * for dealing with snapshots, by removing the snapshot directory, or
1153 * in handling vendor migrations where files have been restored with
1154 * a vendor product into a subdirectory.
1156 void strip_path(FF_PKT *ff_pkt)
1158 if (!(ff_pkt->flags & FO_STRIPPATH) || ff_pkt->strip_path <= 0) {
1159 Dmsg1(200, "No strip for %s\n", ff_pkt->fname);
1162 if (!ff_pkt->fname_save) {
1163 ff_pkt->fname_save = get_pool_memory(PM_FNAME);
1164 ff_pkt->link_save = get_pool_memory(PM_FNAME);
1166 pm_strcpy(ff_pkt->fname_save, ff_pkt->fname);
1167 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1168 pm_strcpy(ff_pkt->link_save, ff_pkt->link);
1169 Dmsg2(500, "strcpy link_save=%d link=%d\n", strlen(ff_pkt->link_save),
1170 strlen(ff_pkt->link));
1171 sm_check(__FILE__, __LINE__, true);
1175 * Strip path. If it doesn't succeed put it back. If
1176 * it does, and there is a different link string,
1177 * attempt to strip the link. If it fails, back them
1179 * Do not strip symlinks.
1180 * I.e. if either stripping fails don't strip anything.
1182 if (!do_strip(ff_pkt->strip_path, ff_pkt->fname)) {
1183 unstrip_path(ff_pkt);
1186 /* Strip links but not symlinks */
1187 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1188 if (!do_strip(ff_pkt->strip_path, ff_pkt->link)) {
1189 unstrip_path(ff_pkt);
1194 Dmsg3(100, "fname=%s stripped=%s link=%s\n", ff_pkt->fname_save, ff_pkt->fname,
1198 void unstrip_path(FF_PKT *ff_pkt)
1200 if (!(ff_pkt->flags & FO_STRIPPATH) || ff_pkt->strip_path <= 0) {
1203 strcpy(ff_pkt->fname, ff_pkt->fname_save);
1204 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1205 Dmsg2(500, "strcpy link=%s link_save=%s\n", ff_pkt->link,
1207 strcpy(ff_pkt->link, ff_pkt->link_save);
1208 Dmsg2(500, "strcpy link=%d link_save=%d\n", strlen(ff_pkt->link),
1209 strlen(ff_pkt->link_save));
1210 sm_check(__FILE__, __LINE__, true);