2 Bacula® - The Network Backup Solution
4 Copyright (C) 2000-2008 Free Software Foundation Europe e.V.
6 The main author of Bacula is Kern Sibbald, with contributions from
7 many others, a complete list can be found in the file AUTHORS.
8 This program is Free Software; you can redistribute it and/or
9 modify it under the terms of version two of the GNU General Public
10 License as published by the Free Software Foundation and included
13 This program is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
23 Bacula® is a registered trademark of John Walker.
24 The licensor of Bacula is the Free Software Foundation Europe
25 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
26 Switzerland, email:ftf@fsfeurope.org.
29 * Bacula File Daemon backup.c send file attributes and data
30 * to the Storage daemon.
32 * Kern Sibbald, March MM
41 /* Forward referenced functions */
42 int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level);
43 static void strip_path(FF_PKT *ff_pkt);
44 static void unstrip_path(FF_PKT *ff_pkt);
45 static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, DIGEST *signature_digest);
46 static bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream);
47 static bool read_and_send_acl(JCR *jcr, int acltype, int stream);
48 static bool crypto_session_start(JCR *jcr);
49 static void crypto_session_end(JCR *jcr);
50 static bool crypto_session_send(JCR *jcr, BSOCK *sd);
53 * Find all the requested files and send them
54 * to the Storage daemon.
56 * Note, we normally carry on a one-way
57 * conversation from this point on with the SD, simply blasting
58 * data to him. To properly know what is going on, we
59 * also run a "heartbeat" monitor which reads the socket and
60 * reacts accordingly (at the moment it has nothing to do
61 * except echo the heartbeat to the Director).
64 bool blast_data_to_storage_daemon(JCR *jcr, char *addr)
68 // TODO landonf: Allow user to specify encryption algorithm
70 sd = jcr->store_bsock;
72 set_jcr_job_status(jcr, JS_Running);
74 Dmsg1(300, "bfiled: opened data connection %d to stored\n", sd->m_fd);
77 CLIENT *client = (CLIENT *)GetNextRes(R_CLIENT, NULL);
81 buf_size = client->max_network_buffer_size;
83 buf_size = 0; /* use default */
85 if (!bnet_set_buffer_size(sd, buf_size, BNET_SETBUF_WRITE)) {
86 set_jcr_job_status(jcr, JS_ErrorTerminated);
87 Jmsg(jcr, M_FATAL, 0, _("Cannot set buffer size FD->SD.\n"));
91 jcr->buf_size = sd->msglen;
92 /* Adjust for compression so that output buffer is
93 * 12 bytes + 0.1% larger than input buffer plus 18 bytes.
94 * This gives a bit extra plus room for the sparse addr if any.
95 * Note, we adjust the read size to be smaller so that the
96 * same output buffer can be used without growing it.
98 * The zlib compression workset is initialized here to minimise
99 * the "per file" load. The jcr member is only set, if the init was successful.
101 jcr->compress_buf_size = jcr->buf_size + ((jcr->buf_size+999) / 1000) + 30;
102 jcr->compress_buf = get_memory(jcr->compress_buf_size);
105 z_stream *pZlibStream = (z_stream*)malloc(sizeof(z_stream));
107 pZlibStream->zalloc = Z_NULL;
108 pZlibStream->zfree = Z_NULL;
109 pZlibStream->opaque = Z_NULL;
110 pZlibStream->state = Z_NULL;
112 if (deflateInit(pZlibStream, Z_DEFAULT_COMPRESSION) == Z_OK) {
113 jcr->pZLIB_compress_workset = pZlibStream;
120 if (!crypto_session_start(jcr)) {
124 Dmsg1(300, "set_find_options ff=%p\n", jcr->ff);
125 set_find_options((FF_PKT *)jcr->ff, jcr->incremental, jcr->mtime);
126 Dmsg0(300, "start find files\n");
128 start_heartbeat_monitor(jcr);
130 jcr->acl_text = get_pool_memory(PM_MESSAGE);
132 /* Subroutine save_file() is called for each file */
133 if (!find_files(jcr, (FF_PKT *)jcr->ff, save_file, plugin_save)) {
134 ok = false; /* error */
135 set_jcr_job_status(jcr, JS_ErrorTerminated);
138 free_pool_memory(jcr->acl_text);
140 stop_heartbeat_monitor(jcr);
142 sd->signal(BNET_EOD); /* end of sending data */
148 if (jcr->compress_buf) {
149 free_pool_memory(jcr->compress_buf);
150 jcr->compress_buf = NULL;
152 if (jcr->pZLIB_compress_workset) {
153 /* Free the zlib stream */
155 deflateEnd((z_stream *)jcr->pZLIB_compress_workset);
157 free (jcr->pZLIB_compress_workset);
158 jcr->pZLIB_compress_workset = NULL;
160 crypto_session_end(jcr);
163 Dmsg1(100, "end blast_data ok=%d\n", ok);
167 static bool crypto_session_start(JCR *jcr)
169 crypto_cipher_t cipher = CRYPTO_CIPHER_AES_128_CBC;
172 * Create encryption session data and a cached, DER-encoded session data
173 * structure. We use a single session key for each backup, so we'll encode
174 * the session data only once.
176 if (jcr->crypto.pki_encrypt) {
179 /* Create per-job session encryption context */
180 jcr->crypto.pki_session = crypto_session_new(cipher, jcr->crypto.pki_recipients);
182 /* Get the session data size */
183 if (!crypto_session_encode(jcr->crypto.pki_session, (uint8_t *)0, &size)) {
184 Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n"));
188 /* Allocate buffer */
189 jcr->crypto.pki_session_encoded = get_memory(size);
191 /* Encode session data */
192 if (!crypto_session_encode(jcr->crypto.pki_session, (uint8_t *)jcr->crypto.pki_session_encoded, &size)) {
193 Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n"));
197 /* ... and store the encoded size */
198 jcr->crypto.pki_session_encoded_size = size;
200 /* Allocate the encryption/decryption buffer */
201 jcr->crypto.crypto_buf = get_memory(CRYPTO_CIPHER_MAX_BLOCK_SIZE);
206 static void crypto_session_end(JCR *jcr)
208 if (jcr->crypto.crypto_buf) {
209 free_pool_memory(jcr->crypto.crypto_buf);
210 jcr->crypto.crypto_buf = NULL;
212 if (jcr->crypto.pki_session) {
213 crypto_session_free(jcr->crypto.pki_session);
215 if (jcr->crypto.pki_session_encoded) {
216 free_pool_memory(jcr->crypto.pki_session_encoded);
217 jcr->crypto.pki_session_encoded = NULL;
221 static bool crypto_session_send(JCR *jcr, BSOCK *sd)
225 /* Send our header */
226 Dmsg2(100, "Send hdr fi=%ld stream=%d\n", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA);
227 sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA);
230 sd->msg = jcr->crypto.pki_session_encoded;
231 sd->msglen = jcr->crypto.pki_session_encoded_size;
232 jcr->JobBytes += sd->msglen;
234 Dmsg1(100, "Send data len=%d\n", sd->msglen);
237 sd->signal(BNET_EOD);
243 * Called here by find() for each file included.
244 * This is a callback. The original is find_files() above.
246 * Send the file and its data to the Storage daemon.
250 * -1 to ignore file/directory (not used here)
252 int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level)
254 bool do_read = false;
255 int stat, data_stream;
257 DIGEST *digest = NULL;
258 DIGEST *signing_digest = NULL;
259 int digest_stream = STREAM_NONE;
260 SIGNATURE *sig = NULL;
261 bool has_file_data = false;
262 // TODO landonf: Allow the user to specify the digest algorithm
264 crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA256;
266 crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA1;
268 BSOCK *sd = jcr->store_bsock;
270 if (job_canceled(jcr)) {
274 jcr->num_files_examined++; /* bump total file count */
276 switch (ff_pkt->type) {
277 case FT_LNKSAVED: /* Hard linked, file already saved */
278 Dmsg2(130, "FT_LNKSAVED hard link: %s => %s\n", ff_pkt->fname, ff_pkt->link);
281 Dmsg1(130, "FT_REGE saving: %s\n", ff_pkt->fname);
282 has_file_data = true;
285 Dmsg1(130, "FT_REG saving: %s\n", ff_pkt->fname);
286 has_file_data = true;
289 Dmsg2(130, "FT_LNK saving: %s -> %s\n", ff_pkt->fname, ff_pkt->link);
292 jcr->num_files_examined--; /* correct file count */
293 return 1; /* not used */
295 Jmsg(jcr, M_INFO, 1, _(" Recursion turned off. Will not descend from %s into %s\n"),
296 ff_pkt->top_fname, ff_pkt->fname);
297 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
300 /* Suppress message for /dev filesystems */
301 if (strncmp(ff_pkt->fname, "/dev/", 5) != 0) {
302 Jmsg(jcr, M_INFO, 1, _(" %s is a different filesystem. Will not descend from %s into %s\n"),
303 ff_pkt->fname, ff_pkt->top_fname, ff_pkt->fname);
305 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
308 Jmsg(jcr, M_INFO, 1, _(" Disallowed filesystem. Will not descend from %s into %s\n"),
309 ff_pkt->top_fname, ff_pkt->fname);
310 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
313 Jmsg(jcr, M_INFO, 1, _(" Disallowed drive type. Will not descend into %s\n"),
318 Dmsg1(130, "FT_DIREND: %s\n", ff_pkt->link);
321 Dmsg1(130, "FT_SPEC saving: %s\n", ff_pkt->fname);
322 if (S_ISSOCK(ff_pkt->statp.st_mode)) {
323 Jmsg(jcr, M_SKIPPED, 1, _(" Socket file skipped: %s\n"), ff_pkt->fname);
328 Dmsg1(130, "FT_RAW saving: %s\n", ff_pkt->fname);
329 has_file_data = true;
332 Dmsg1(130, "FT_FIFO saving: %s\n", ff_pkt->fname);
336 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not access %s: ERR=%s\n"), ff_pkt->fname,
337 be.bstrerror(ff_pkt->ff_errno));
343 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not follow link %s: ERR=%s\n"),
344 ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
350 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not stat %s: ERR=%s\n"), ff_pkt->fname,
351 be.bstrerror(ff_pkt->ff_errno));
357 Jmsg(jcr, M_SKIPPED, 1, _(" Unchanged file skipped: %s\n"), ff_pkt->fname);
360 Jmsg(jcr, M_NOTSAVED, 0, _(" Archive file not saved: %s\n"), ff_pkt->fname);
364 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not open directory %s: ERR=%s\n"),
365 ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
370 Jmsg(jcr, M_NOTSAVED, 0, _(" Unknown file type %d; not saved: %s\n"),
371 ff_pkt->type, ff_pkt->fname);
376 Dmsg1(130, "bfiled: sending %s to stored\n", ff_pkt->fname);
378 /* Digests and encryption are only useful if there's file data */
381 * Setup for digest handling. If this fails, the digest will be set to NULL
382 * and not used. Note, the digest (file hash) can be any one of the four
385 * The signing digest is a single algorithm depending on
386 * whether or not we have SHA2.
387 * ****FIXME**** the signing algoritm should really be
388 * determined a different way!!!!!! What happens if
389 * sha2 was available during backup but not restore?
391 if (ff_pkt->flags & FO_MD5) {
392 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_MD5);
393 digest_stream = STREAM_MD5_DIGEST;
395 } else if (ff_pkt->flags & FO_SHA1) {
396 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA1);
397 digest_stream = STREAM_SHA1_DIGEST;
399 } else if (ff_pkt->flags & FO_SHA256) {
400 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA256);
401 digest_stream = STREAM_SHA256_DIGEST;
403 } else if (ff_pkt->flags & FO_SHA512) {
404 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA512);
405 digest_stream = STREAM_SHA512_DIGEST;
408 /* Did digest initialization fail? */
409 if (digest_stream != STREAM_NONE && digest == NULL) {
410 Jmsg(jcr, M_WARNING, 0, _("%s digest initialization failed\n"),
411 stream_to_ascii(digest_stream));
415 * Set up signature digest handling. If this fails, the signature digest will be set to
418 // TODO landonf: We should really only calculate the digest once, for both verification and signing.
419 if (jcr->crypto.pki_sign) {
420 signing_digest = crypto_digest_new(jcr, signing_algorithm);
422 /* Full-stop if a failure occurred initializing the signature digest */
423 if (signing_digest == NULL) {
424 Jmsg(jcr, M_NOTSAVED, 0, _("%s signature digest initialization failed\n"),
425 stream_to_ascii(signing_algorithm));
431 /* Enable encryption */
432 if (jcr->crypto.pki_encrypt) {
433 ff_pkt->flags |= FO_ENCRYPT;
437 /* Initialize the file descriptor we use for data and other streams. */
439 if (ff_pkt->flags & FO_PORTABLE) {
440 set_portable_backup(&ff_pkt->bfd); /* disable Win32 BackupRead() */
442 if (ff_pkt->cmd_plugin) {
443 if (!set_cmd_plugin(&ff_pkt->bfd, jcr)) {
446 send_plugin_name(jcr, sd, true); /* signal start of plugin data */
449 /* Send attributes -- must be done after binit() */
450 if (!encode_and_send_attributes(jcr, ff_pkt, data_stream)) {
454 /* Set up the encryption context and send the session data to the SD */
455 if (has_file_data && jcr->crypto.pki_encrypt) {
456 if (!crypto_session_send(jcr, sd)) {
462 * Open any file with data that we intend to save, then save it.
464 * Note, if is_win32_backup, we must open the Directory so that
465 * the BackupRead will save its permissions and ownership streams.
467 if (ff_pkt->type != FT_LNKSAVED && S_ISREG(ff_pkt->statp.st_mode)) {
469 do_read = !is_portable_backup(&ff_pkt->bfd) || ff_pkt->statp.st_size > 0;
471 do_read = ff_pkt->statp.st_size > 0;
473 } else if (ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO ||
474 ff_pkt->type == FT_REPARSE ||
475 (!is_portable_backup(&ff_pkt->bfd) && ff_pkt->type == FT_DIREND)) {
478 if (ff_pkt->cmd_plugin) {
482 Dmsg1(100, "do_read=%d\n", do_read);
486 if (ff_pkt->type == FT_FIFO) {
487 tid = start_thread_timer(jcr, pthread_self(), 60);
491 int noatime = ff_pkt->flags & FO_NOATIME ? O_NOATIME : 0;
492 ff_pkt->bfd.reparse_point = ff_pkt->type == FT_REPARSE;
493 if (bopen(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY | noatime, 0) < 0) {
494 ff_pkt->ff_errno = errno;
496 Jmsg(jcr, M_NOTSAVED, 0, _(" Cannot open %s: ERR=%s.\n"), ff_pkt->fname,
500 stop_thread_timer(tid);
506 stop_thread_timer(tid);
510 stat = send_data(jcr, data_stream, ff_pkt, digest, signing_digest);
512 if (ff_pkt->flags & FO_CHKCHANGES) {
513 has_file_changed(jcr, ff_pkt);
516 bclose(&ff_pkt->bfd);
523 #ifdef HAVE_DARWIN_OS
524 /* Regular files can have resource forks and Finder Info */
525 if (ff_pkt->type != FT_LNKSAVED && (S_ISREG(ff_pkt->statp.st_mode) &&
526 ff_pkt->flags & FO_HFSPLUS)) {
527 if (ff_pkt->hfsinfo.rsrclength > 0) {
530 if (!bopen_rsrc(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0) < 0) {
531 ff_pkt->ff_errno = errno;
533 Jmsg(jcr, M_NOTSAVED, -1, _(" Cannot open resource fork for %s: ERR=%s.\n"),
534 ff_pkt->fname, be.bstrerror());
536 if (is_bopen(&ff_pkt->bfd)) {
537 bclose(&ff_pkt->bfd);
541 flags = ff_pkt->flags;
542 ff_pkt->flags &= ~(FO_GZIP|FO_SPARSE);
543 if (flags & FO_ENCRYPT) {
544 rsrc_stream = STREAM_ENCRYPTED_MACOS_FORK_DATA;
546 rsrc_stream = STREAM_MACOS_FORK_DATA;
548 stat = send_data(jcr, rsrc_stream, ff_pkt, digest, signing_digest);
549 ff_pkt->flags = flags;
550 bclose(&ff_pkt->bfd);
556 Dmsg1(300, "Saving Finder Info for \"%s\"\n", ff_pkt->fname);
557 sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_HFSPLUS_ATTRIBUTES);
558 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
559 memcpy(sd->msg, ff_pkt->hfsinfo.fndrinfo, 32);
562 crypto_digest_update(digest, (uint8_t *)sd->msg, sd->msglen);
564 if (signing_digest) {
565 crypto_digest_update(signing_digest, (uint8_t *)sd->msg, sd->msglen);
568 sd->signal(BNET_EOD);
572 if (ff_pkt->flags & FO_ACL) {
573 /* Read access ACLs for files, dirs and links */
574 if (!read_and_send_acl(jcr, BACL_TYPE_ACCESS, STREAM_UNIX_ACCESS_ACL)) {
577 /* Directories can have default ACLs too */
578 if (ff_pkt->type == FT_DIREND && (BACL_CAP & BACL_CAP_DEFAULTS_DIR)) {
579 if (!read_and_send_acl(jcr, BACL_TYPE_DEFAULT, STREAM_UNIX_DEFAULT_ACL)) {
585 /* Terminate the signing digest and send it to the Storage daemon */
586 if (signing_digest) {
589 if ((sig = crypto_sign_new(jcr)) == NULL) {
590 Jmsg(jcr, M_FATAL, 0, _("Failed to allocate memory for crypto signature.\n"));
594 if (!crypto_sign_add_signer(sig, signing_digest, jcr->crypto.pki_keypair)) {
595 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
599 /* Get signature size */
600 if (!crypto_sign_encode(sig, NULL, &size)) {
601 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
605 /* Grow the bsock buffer to fit our message if necessary */
606 if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
607 sd->msg = realloc_pool_memory(sd->msg, size);
610 /* Send our header */
611 sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_SIGNED_DIGEST);
612 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
614 /* Encode signature data */
615 if (!crypto_sign_encode(sig, (uint8_t *)sd->msg, &size)) {
616 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
622 sd->signal(BNET_EOD); /* end of checksum */
625 /* Terminate any digest and send it to Storage daemon */
629 sd->fsend("%ld %d 0", jcr->JobFiles, digest_stream);
630 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
632 size = CRYPTO_DIGEST_MAX_SIZE;
634 /* Grow the bsock buffer to fit our message if necessary */
635 if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
636 sd->msg = realloc_pool_memory(sd->msg, size);
639 if (!crypto_digest_finalize(digest, (uint8_t *)sd->msg, &size)) {
640 Jmsg(jcr, M_FATAL, 0, _("An error occurred finalizing signing the stream.\n"));
646 sd->signal(BNET_EOD); /* end of checksum */
648 if (ff_pkt->cmd_plugin) {
649 send_plugin_name(jcr, sd, false); /* signal end of plugin data */
653 rtnstat = 1; /* good return */
657 crypto_digest_free(digest);
659 if (signing_digest) {
660 crypto_digest_free(signing_digest);
663 crypto_sign_free(sig);
669 * Send data read from an already open file descriptor.
671 * We return 1 on sucess and 0 on errors.
674 * We use ff_pkt->statp.st_size when FO_SPARSE to know when to stop
676 * Currently this is not a problem as the only other stream, resource forks,
677 * are not handled as sparse files.
679 int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest,
680 DIGEST *signing_digest)
682 BSOCK *sd = jcr->store_bsock;
683 uint64_t fileAddr = 0; /* file address */
685 int32_t rsize = jcr->buf_size; /* read buffer size */
687 CIPHER_CONTEXT *cipher_ctx = NULL; /* Quell bogus uninitialized warnings */
688 const uint8_t *cipher_input;
689 uint32_t cipher_input_len;
690 uint32_t cipher_block_size;
691 uint32_t encrypted_len;
692 #ifdef FD_NO_SEND_TEST
697 rbuf = sd->msg; /* read buffer */
698 wbuf = sd->msg; /* write buffer */
699 cipher_input = (uint8_t *)rbuf; /* encrypt uncompressed data */
701 Dmsg1(300, "Saving data, type=%d\n", ff_pkt->type);
704 uLong compress_len = 0;
705 uLong max_compress_len = 0;
706 const Bytef *cbuf = NULL;
709 if (ff_pkt->flags & FO_GZIP) {
710 if (ff_pkt->flags & FO_SPARSE) {
711 cbuf = (Bytef *)jcr->compress_buf + SPARSE_FADDR_SIZE;
712 max_compress_len = jcr->compress_buf_size - SPARSE_FADDR_SIZE;
714 cbuf = (Bytef *)jcr->compress_buf;
715 max_compress_len = jcr->compress_buf_size; /* set max length */
717 wbuf = jcr->compress_buf; /* compressed output here */
718 cipher_input = (uint8_t *)jcr->compress_buf; /* encrypt compressed data */
721 * Only change zlib parameters if there is no pending operation.
722 * This should never happen as deflatereset is called after each
726 if (((z_stream*)jcr->pZLIB_compress_workset)->total_in == 0) {
727 /* set gzip compression level - must be done per file */
728 if ((zstat=deflateParams((z_stream*)jcr->pZLIB_compress_workset,
729 ff_pkt->GZIP_level, Z_DEFAULT_STRATEGY)) != Z_OK) {
730 Jmsg(jcr, M_FATAL, 0, _("Compression deflateParams error: %d\n"), zstat);
731 set_jcr_job_status(jcr, JS_ErrorTerminated);
737 const uint32_t max_compress_len = 0;
740 if (ff_pkt->flags & FO_ENCRYPT) {
741 if (ff_pkt->flags & FO_SPARSE) {
742 Jmsg0(jcr, M_FATAL, 0, _("Encrypting sparse data not supported.\n"));
745 /* Allocate the cipher context */
746 if ((cipher_ctx = crypto_cipher_new(jcr->crypto.pki_session, true,
747 &cipher_block_size)) == NULL) {
748 /* Shouldn't happen! */
749 Jmsg0(jcr, M_FATAL, 0, _("Failed to initialize encryption context.\n"));
754 * Grow the crypto buffer, if necessary.
755 * crypto_cipher_update() will buffer up to (cipher_block_size - 1).
756 * We grow crypto_buf to the maximum number of blocks that
757 * could be returned for the given read buffer size.
758 * (Using the larger of either rsize or max_compress_len)
760 jcr->crypto.crypto_buf = check_pool_memory_size(jcr->crypto.crypto_buf,
761 (MAX(rsize + (int)sizeof(uint32_t), (int32_t)max_compress_len) +
762 cipher_block_size - 1) / cipher_block_size * cipher_block_size);
764 wbuf = jcr->crypto.crypto_buf; /* Encrypted, possibly compressed output here. */
768 * Send Data header to Storage daemon
769 * <file-index> <stream> <info>
771 if (!sd->fsend("%ld %d 0", jcr->JobFiles, stream)) {
772 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
776 Dmsg1(300, ">stored: datahdr %s\n", sd->msg);
779 * Make space at beginning of buffer for fileAddr because this
780 * same buffer will be used for writing if compression is off.
782 if (ff_pkt->flags & FO_SPARSE) {
783 rbuf += SPARSE_FADDR_SIZE;
784 rsize -= SPARSE_FADDR_SIZE;
785 #ifdef HAVE_FREEBSD_OS
787 * To read FreeBSD partitions, the read size must be
790 rsize = (rsize/512) * 512;
794 /* a RAW device read on win32 only works if the buffer is a multiple of 512 */
796 if (S_ISBLK(ff_pkt->statp.st_mode))
797 rsize = (rsize/512) * 512;
803 while ((sd->msglen=(uint32_t)bread(&ff_pkt->bfd, rbuf, rsize)) > 0) {
805 /* Check for sparse blocks */
806 if (ff_pkt->flags & FO_SPARSE) {
808 bool haveBlock = true;
809 if (sd->msglen == rsize &&
810 fileAddr+sd->msglen < (uint64_t)ff_pkt->statp.st_size ||
811 ((ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO) &&
812 (uint64_t)ff_pkt->statp.st_size == 0)) {
813 haveBlock = !is_buf_zero(rbuf, rsize);
816 ser_begin(wbuf, SPARSE_FADDR_SIZE);
817 ser_uint64(fileAddr); /* store fileAddr in begin of buffer */
819 fileAddr += sd->msglen; /* update file address */
821 continue; /* skip block of zeros */
825 jcr->ReadBytes += sd->msglen; /* count bytes read */
827 /* Uncompressed cipher input length */
828 cipher_input_len = sd->msglen;
830 /* Update checksum if requested */
832 crypto_digest_update(digest, (uint8_t *)rbuf, sd->msglen);
835 /* Update signing digest if requested */
836 if (signing_digest) {
837 crypto_digest_update(signing_digest, (uint8_t *)rbuf, sd->msglen);
841 /* Do compression if turned on */
842 if (ff_pkt->flags & FO_GZIP && jcr->pZLIB_compress_workset) {
843 Dmsg3(400, "cbuf=0x%x rbuf=0x%x len=%u\n", cbuf, rbuf, sd->msglen);
845 ((z_stream*)jcr->pZLIB_compress_workset)->next_in = (Bytef *)rbuf;
846 ((z_stream*)jcr->pZLIB_compress_workset)->avail_in = sd->msglen;
847 ((z_stream*)jcr->pZLIB_compress_workset)->next_out = (Bytef *)cbuf;
848 ((z_stream*)jcr->pZLIB_compress_workset)->avail_out = max_compress_len;
850 if ((zstat=deflate((z_stream*)jcr->pZLIB_compress_workset, Z_FINISH)) != Z_STREAM_END) {
851 Jmsg(jcr, M_FATAL, 0, _("Compression deflate error: %d\n"), zstat);
852 set_jcr_job_status(jcr, JS_ErrorTerminated);
855 compress_len = ((z_stream*)jcr->pZLIB_compress_workset)->total_out;
856 /* reset zlib stream to be able to begin from scratch again */
857 if ((zstat=deflateReset((z_stream*)jcr->pZLIB_compress_workset)) != Z_OK) {
858 Jmsg(jcr, M_FATAL, 0, _("Compression deflateReset error: %d\n"), zstat);
859 set_jcr_job_status(jcr, JS_ErrorTerminated);
863 Dmsg2(400, "compressed len=%d uncompressed len=%d\n", compress_len,
866 sd->msglen = compress_len; /* set compressed length */
867 cipher_input_len = compress_len;
871 * Note, here we prepend the current record length to the beginning
872 * of the encrypted data. This is because both sparse and compression
873 * restore handling want records returned to them with exactly the
874 * same number of bytes that were processed in the backup handling.
875 * That is, both are block filters rather than a stream. When doing
876 * compression, the compression routines may buffer data, so that for
877 * any one record compressed, when it is decompressed the same size
878 * will not be obtained. Of course, the buffered data eventually comes
879 * out in subsequent crypto_cipher_update() calls or at least
880 * when crypto_cipher_finalize() is called. Unfortunately, this
881 * "feature" of encryption enormously complicates the restore code.
883 if (ff_pkt->flags & FO_ENCRYPT) {
884 uint32_t initial_len = 0;
887 if (ff_pkt->flags & FO_SPARSE) {
888 cipher_input_len += SPARSE_FADDR_SIZE;
891 /* Encrypt the length of the input block */
892 uint8_t packet_len[sizeof(uint32_t)];
894 ser_begin(packet_len, sizeof(uint32_t));
895 ser_uint32(cipher_input_len); /* store data len in begin of buffer */
896 Dmsg1(20, "Encrypt len=%d\n", cipher_input_len);
898 if (!crypto_cipher_update(cipher_ctx, packet_len, sizeof(packet_len),
899 (uint8_t *)jcr->crypto.crypto_buf, &initial_len)) {
900 /* Encryption failed. Shouldn't happen. */
901 Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
905 /* Encrypt the input block */
906 if (crypto_cipher_update(cipher_ctx, cipher_input, cipher_input_len,
907 (uint8_t *)&jcr->crypto.crypto_buf[initial_len], &encrypted_len)) {
908 if ((initial_len + encrypted_len) == 0) {
909 /* No full block of data available, read more data */
912 Dmsg2(400, "encrypted len=%d unencrypted len=%d\n", encrypted_len,
914 sd->msglen = initial_len + encrypted_len; /* set encrypted length */
916 /* Encryption failed. Shouldn't happen. */
917 Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
922 /* Send the buffer to the Storage daemon */
923 if (ff_pkt->flags & FO_SPARSE) {
924 sd->msglen += SPARSE_FADDR_SIZE; /* include fileAddr in size */
926 sd->msg = wbuf; /* set correct write buffer */
928 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
932 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
934 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
935 sd->msg = msgsave; /* restore read buffer */
937 } /* end while read file data */
939 if (sd->msglen < 0) { /* error */
941 Jmsg(jcr, M_ERROR, 0, _("Read error on file %s. ERR=%s\n"),
942 ff_pkt->fname, be.bstrerror(ff_pkt->bfd.berrno));
943 if (jcr->Errors++ > 1000) { /* insanity check */
944 Jmsg(jcr, M_FATAL, 0, _("Too many errors.\n"));
946 } else if (ff_pkt->flags & FO_ENCRYPT) {
948 * For encryption, we must call finalize to push out any
951 if (!crypto_cipher_finalize(cipher_ctx, (uint8_t *)jcr->crypto.crypto_buf,
953 /* Padding failed. Shouldn't happen. */
954 Jmsg(jcr, M_FATAL, 0, _("Encryption padding error\n"));
958 /* Note, on SSL pre-0.9.7, there is always some output */
959 if (encrypted_len > 0) {
960 sd->msglen = encrypted_len; /* set encrypted length */
961 sd->msg = jcr->crypto.crypto_buf; /* set correct write buffer */
963 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
967 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
968 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
969 sd->msg = msgsave; /* restore bnet buffer */
973 if (!sd->signal(BNET_EOD)) { /* indicate end of file data */
974 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
979 /* Free the cipher context */
981 crypto_cipher_free(cipher_ctx);
986 /* Free the cipher context */
988 crypto_cipher_free(cipher_ctx);
991 sd->msg = msgsave; /* restore bnet buffer */
997 * Read and send an ACL for the last encountered file.
999 static bool read_and_send_acl(JCR *jcr, int acltype, int stream)
1002 BSOCK *sd = jcr->store_bsock;
1005 #ifdef FD_NO_SEND_TEST
1009 len = bacl_get(jcr, acltype);
1011 Jmsg1(jcr, M_WARNING, 0, _("Error reading ACL of %s\n"), jcr->last_fname);
1015 return true; /* no ACL */
1019 if (!sd->fsend("%ld %d 0", jcr->JobFiles, stream)) {
1020 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1025 /* Send the buffer to the storage deamon */
1026 Dmsg2(400, "Backing up ACL type 0x%2x <%s>\n", acltype, jcr->acl_text);
1028 sd->msg = jcr->acl_text;
1029 sd->msglen = len + 1;
1033 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1038 jcr->JobBytes += sd->msglen;
1040 if (!sd->signal(BNET_EOD)) {
1041 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1046 Dmsg1(200, "ACL of file: %s successfully backed up!\n", jcr->last_fname);
1051 static bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream)
1053 BSOCK *sd = jcr->store_bsock;
1054 char attribs[MAXSTRING];
1055 char attribsEx[MAXSTRING];
1058 #ifdef FD_NO_SEND_TEST
1062 Dmsg1(300, "encode_and_send_attrs fname=%s\n", ff_pkt->fname);
1063 /* Find what data stream we will use, then encode the attributes */
1064 if ((data_stream = select_data_stream(ff_pkt)) == STREAM_NONE) {
1065 /* This should not happen */
1066 Jmsg0(jcr, M_FATAL, 0, _("Invalid file flags, no supported data stream type.\n"));
1069 encode_stat(attribs, ff_pkt, data_stream);
1071 /* Now possibly extend the attributes */
1072 attr_stream = encode_attribsEx(jcr, attribsEx, ff_pkt);
1074 Dmsg3(300, "File %s\nattribs=%s\nattribsEx=%s\n", ff_pkt->fname, attribs, attribsEx);
1077 jcr->JobFiles++; /* increment number of files sent */
1078 ff_pkt->FileIndex = jcr->JobFiles; /* return FileIndex */
1079 pm_strcpy(jcr->last_fname, ff_pkt->fname);
1083 * Send Attributes header to Storage daemon
1084 * <file-index> <stream> <info>
1086 if (!sd->fsend("%ld %d 0", jcr->JobFiles, attr_stream)) {
1087 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1091 Dmsg1(300, ">stored: attrhdr %s\n", sd->msg);
1094 * Send file attributes to Storage daemon
1097 * Filename (full path)
1098 * Encoded attributes
1099 * Link name (if type==FT_LNK or FT_LNKSAVED)
1100 * Encoded extended-attributes (for Win32)
1102 * For a directory, link is the same as fname, but with trailing
1103 * slash. For a linked file, link is the link.
1106 if (ff_pkt->type == FT_LNK || ff_pkt->type == FT_LNKSAVED) {
1107 Dmsg2(300, "Link %s to %s\n", ff_pkt->fname, ff_pkt->link);
1108 stat = sd->fsend("%ld %d %s%c%s%c%s%c%s%c", jcr->JobFiles,
1109 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, ff_pkt->link, 0,
1111 } else if (ff_pkt->type == FT_DIREND || ff_pkt->type == FT_REPARSE) {
1112 /* Here link is the canonical filename (i.e. with trailing slash) */
1113 stat = sd->fsend("%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
1114 ff_pkt->type, ff_pkt->link, 0, attribs, 0, 0, attribsEx, 0);
1116 stat = sd->fsend("%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
1117 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, 0, attribsEx, 0);
1119 unstrip_path(ff_pkt);
1121 Dmsg2(300, ">stored: attr len=%d: %s\n", sd->msglen, sd->msg);
1123 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1127 sd->signal(BNET_EOD); /* indicate end of attributes data */
1132 * Do in place strip of path
1134 static bool do_strip(int count, char *in)
1140 /* Copy to first path separator -- Win32 might have c: ... */
1141 while (*in && !IsPathSeparator(*in)) {
1145 numsep++; /* one separator seen */
1146 for (stripped=0; stripped<count && *in; stripped++) {
1147 while (*in && !IsPathSeparator(*in)) {
1148 in++; /* skip chars */
1151 numsep++; /* count separators seen */
1152 in++; /* skip separator */
1156 while (*in) { /* copy to end */
1157 if (IsPathSeparator(*in)) {
1163 Dmsg4(500, "stripped=%d count=%d numsep=%d sep>count=%d\n",
1164 stripped, count, numsep, numsep>count);
1165 return stripped==count && numsep>count;
1169 * If requested strip leading components of the path
1171 static void strip_path(FF_PKT *ff_pkt)
1173 if (!(ff_pkt->flags & FO_STRIPPATH) || ff_pkt->strip_path <= 0) {
1174 Dmsg1(200, "No strip for %s\n", ff_pkt->fname);
1177 if (!ff_pkt->fname_save) {
1178 ff_pkt->fname_save = get_pool_memory(PM_FNAME);
1179 ff_pkt->link_save = get_pool_memory(PM_FNAME);
1181 pm_strcpy(ff_pkt->fname_save, ff_pkt->fname);
1184 * Strip path. If it doesn't succeed put it back. If
1185 * it does, and there is a different link string,
1186 * attempt to strip the link. If it fails, back them
1188 * Do not strip symlinks.
1189 * I.e. if either stripping fails don't strip anything.
1191 if (do_strip(ff_pkt->strip_path, ff_pkt->fname)) {
1192 /* Strip links but not symlinks */
1193 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1194 pm_strcpy(ff_pkt->link_save, ff_pkt->link);
1195 if (!do_strip(ff_pkt->strip_path, ff_pkt->link)) {
1196 strcpy(ff_pkt->link, ff_pkt->link_save);
1197 strcpy(ff_pkt->fname, ff_pkt->fname_save);
1201 strcpy(ff_pkt->fname, ff_pkt->fname_save);
1203 Dmsg2(200, "fname=%s stripped=%s\n", ff_pkt->fname_save, ff_pkt->fname);
1206 static void unstrip_path(FF_PKT *ff_pkt)
1208 if (!(ff_pkt->flags & FO_STRIPPATH) || ff_pkt->strip_path <= 0) {
1211 strcpy(ff_pkt->fname, ff_pkt->fname_save);
1212 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1213 strcpy(ff_pkt->link, ff_pkt->link_save);