2 Bacula® - The Network Backup Solution
4 Copyright (C) 2000-2008 Free Software Foundation Europe e.V.
6 The main author of Bacula is Kern Sibbald, with contributions from
7 many others, a complete list can be found in the file AUTHORS.
8 This program is Free Software; you can redistribute it and/or
9 modify it under the terms of version two of the GNU General Public
10 License as published by the Free Software Foundation and included
13 This program is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
23 Bacula® is a registered trademark of John Walker.
24 The licensor of Bacula is the Free Software Foundation Europe
25 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
26 Switzerland, email:ftf@fsfeurope.org.
29 * Bacula File Daemon backup.c send file attributes and data
30 * to the Storage daemon.
32 * Kern Sibbald, March MM
41 /* Forward referenced functions */
42 int save_file(FF_PKT *ff_pkt, void *pkt, bool top_level);
43 static void strip_path(FF_PKT *ff_pkt);
44 static void unstrip_path(FF_PKT *ff_pkt);
45 static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, DIGEST *signature_digest);
46 static bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream);
47 static bool read_and_send_acl(JCR *jcr, int acltype, int stream);
48 static bool crypto_session_start(JCR *jcr);
49 static void crypto_session_end(JCR *jcr);
50 static bool crypto_session_send(JCR *jcr, BSOCK *sd);
53 * Find all the requested files and send them
54 * to the Storage daemon.
56 * Note, we normally carry on a one-way
57 * conversation from this point on with the SD, simply blasting
58 * data to him. To properly know what is going on, we
59 * also run a "heartbeat" monitor which reads the socket and
60 * reacts accordingly (at the moment it has nothing to do
61 * except echo the heartbeat to the Director).
64 bool blast_data_to_storage_daemon(JCR *jcr, char *addr)
68 // TODO landonf: Allow user to specify encryption algorithm
70 sd = jcr->store_bsock;
72 set_jcr_job_status(jcr, JS_Running);
74 Dmsg1(300, "bfiled: opened data connection %d to stored\n", sd->m_fd);
77 CLIENT *client = (CLIENT *)GetNextRes(R_CLIENT, NULL);
81 buf_size = client->max_network_buffer_size;
83 buf_size = 0; /* use default */
85 if (!bnet_set_buffer_size(sd, buf_size, BNET_SETBUF_WRITE)) {
86 set_jcr_job_status(jcr, JS_ErrorTerminated);
87 Jmsg(jcr, M_FATAL, 0, _("Cannot set buffer size FD->SD.\n"));
91 jcr->buf_size = sd->msglen;
92 /* Adjust for compression so that output buffer is
93 * 12 bytes + 0.1% larger than input buffer plus 18 bytes.
94 * This gives a bit extra plus room for the sparse addr if any.
95 * Note, we adjust the read size to be smaller so that the
96 * same output buffer can be used without growing it.
98 * The zlib compression workset is initialized here to minimise
99 * the "per file" load. The jcr member is only set, if the init was successful.
101 jcr->compress_buf_size = jcr->buf_size + ((jcr->buf_size+999) / 1000) + 30;
102 jcr->compress_buf = get_memory(jcr->compress_buf_size);
105 z_stream *pZlibStream = (z_stream*)malloc(sizeof(z_stream));
107 pZlibStream->zalloc = Z_NULL;
108 pZlibStream->zfree = Z_NULL;
109 pZlibStream->opaque = Z_NULL;
110 pZlibStream->state = Z_NULL;
112 if (deflateInit(pZlibStream, Z_DEFAULT_COMPRESSION) == Z_OK) {
113 jcr->pZLIB_compress_workset = pZlibStream;
120 if (!crypto_session_start(jcr)) {
124 Dmsg1(300, "set_find_options ff=%p\n", jcr->ff);
125 set_find_options((FF_PKT *)jcr->ff, jcr->incremental, jcr->mtime);
126 Dmsg0(300, "start find files\n");
128 start_heartbeat_monitor(jcr);
130 jcr->acl_text = get_pool_memory(PM_MESSAGE);
132 /* Subroutine save_file() is called for each file */
133 if (!find_files(jcr, (FF_PKT *)jcr->ff, save_file, (void *)jcr)) {
134 ok = false; /* error */
135 set_jcr_job_status(jcr, JS_ErrorTerminated);
138 free_pool_memory(jcr->acl_text);
140 stop_heartbeat_monitor(jcr);
142 sd->signal(BNET_EOD); /* end of sending data */
148 if (jcr->compress_buf) {
149 free_pool_memory(jcr->compress_buf);
150 jcr->compress_buf = NULL;
152 if (jcr->pZLIB_compress_workset) {
153 /* Free the zlib stream */
155 deflateEnd((z_stream *)jcr->pZLIB_compress_workset);
157 free (jcr->pZLIB_compress_workset);
158 jcr->pZLIB_compress_workset = NULL;
160 crypto_session_end(jcr);
163 Dmsg1(100, "end blast_data ok=%d\n", ok);
167 static bool crypto_session_start(JCR *jcr)
169 crypto_cipher_t cipher = CRYPTO_CIPHER_AES_128_CBC;
172 * Create encryption session data and a cached, DER-encoded session data
173 * structure. We use a single session key for each backup, so we'll encode
174 * the session data only once.
176 if (jcr->crypto.pki_encrypt) {
179 /* Create per-job session encryption context */
180 jcr->crypto.pki_session = crypto_session_new(cipher, jcr->crypto.pki_recipients);
182 /* Get the session data size */
183 if (!crypto_session_encode(jcr->crypto.pki_session, (uint8_t *)0, &size)) {
184 Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n"));
188 /* Allocate buffer */
189 jcr->crypto.pki_session_encoded = get_memory(size);
191 /* Encode session data */
192 if (!crypto_session_encode(jcr->crypto.pki_session, (uint8_t *)jcr->crypto.pki_session_encoded, &size)) {
193 Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n"));
197 /* ... and store the encoded size */
198 jcr->crypto.pki_session_encoded_size = size;
200 /* Allocate the encryption/decryption buffer */
201 jcr->crypto.crypto_buf = get_memory(CRYPTO_CIPHER_MAX_BLOCK_SIZE);
206 static void crypto_session_end(JCR *jcr)
208 if (jcr->crypto.crypto_buf) {
209 free_pool_memory(jcr->crypto.crypto_buf);
210 jcr->crypto.crypto_buf = NULL;
212 if (jcr->crypto.pki_session) {
213 crypto_session_free(jcr->crypto.pki_session);
215 if (jcr->crypto.pki_session_encoded) {
216 free_pool_memory(jcr->crypto.pki_session_encoded);
217 jcr->crypto.pki_session_encoded = NULL;
221 static bool crypto_session_send(JCR *jcr, BSOCK *sd)
225 /* Send our header */
226 Dmsg2(100, "Send hdr fi=%ld stream=%d\n", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA);
227 sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA);
230 sd->msg = jcr->crypto.pki_session_encoded;
231 sd->msglen = jcr->crypto.pki_session_encoded_size;
232 jcr->JobBytes += sd->msglen;
234 Dmsg1(100, "Send data len=%d\n", sd->msglen);
237 sd->signal(BNET_EOD);
242 * Called here by find() for each file included.
243 * This is a callback. The original is find_files() above.
245 * Send the file and its data to the Storage daemon.
249 * -1 to ignore file/directory (not used here)
251 int save_file(FF_PKT *ff_pkt, void *vjcr, bool top_level)
253 bool do_read = false;
254 int stat, data_stream;
256 DIGEST *digest = NULL;
257 DIGEST *signing_digest = NULL;
258 int digest_stream = STREAM_NONE;
259 SIGNATURE *sig = NULL;
260 bool has_file_data = false;
261 // TODO landonf: Allow the user to specify the digest algorithm
263 crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA256;
265 crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA1;
267 JCR *jcr = (JCR *)vjcr;
268 BSOCK *sd = jcr->store_bsock;
270 if (job_canceled(jcr)) {
274 jcr->num_files_examined++; /* bump total file count */
276 switch (ff_pkt->type) {
277 case FT_LNKSAVED: /* Hard linked, file already saved */
278 Dmsg2(130, "FT_LNKSAVED hard link: %s => %s\n", ff_pkt->fname, ff_pkt->link);
281 Dmsg1(130, "FT_REGE saving: %s\n", ff_pkt->fname);
282 has_file_data = true;
285 Dmsg1(130, "FT_REG saving: %s\n", ff_pkt->fname);
286 has_file_data = true;
289 Dmsg2(130, "FT_LNK saving: %s -> %s\n", ff_pkt->fname, ff_pkt->link);
292 jcr->num_files_examined--; /* correct file count */
293 return 1; /* not used */
295 Jmsg(jcr, M_INFO, 1, _(" Recursion turned off. Will not descend from %s into %s\n"),
296 ff_pkt->top_fname, ff_pkt->fname);
297 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
300 /* Suppress message for /dev filesystems */
301 if (strncmp(ff_pkt->fname, "/dev/", 5) != 0) {
302 Jmsg(jcr, M_INFO, 1, _(" %s is a different filesystem. Will not descend from %s into %s\n"),
303 ff_pkt->fname, ff_pkt->top_fname, ff_pkt->fname);
305 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
308 Jmsg(jcr, M_INFO, 1, _(" Disallowed filesystem. Will not descend from %s into %s\n"),
309 ff_pkt->top_fname, ff_pkt->fname);
310 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
313 Jmsg(jcr, M_INFO, 1, _(" Disallowed drive type. Will not descend into %s\n"),
318 Dmsg1(130, "FT_DIREND: %s\n", ff_pkt->link);
321 Dmsg1(130, "FT_SPEC saving: %s\n", ff_pkt->fname);
322 if (S_ISSOCK(ff_pkt->statp.st_mode)) {
323 Jmsg(jcr, M_SKIPPED, 1, _(" Socket file skipped: %s\n"), ff_pkt->fname);
328 Dmsg1(130, "FT_RAW saving: %s\n", ff_pkt->fname);
329 has_file_data = true;
332 Dmsg1(130, "FT_FIFO saving: %s\n", ff_pkt->fname);
336 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not access %s: ERR=%s\n"), ff_pkt->fname,
337 be.bstrerror(ff_pkt->ff_errno));
343 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not follow link %s: ERR=%s\n"),
344 ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
350 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not stat %s: ERR=%s\n"), ff_pkt->fname,
351 be.bstrerror(ff_pkt->ff_errno));
357 Jmsg(jcr, M_SKIPPED, 1, _(" Unchanged file skipped: %s\n"), ff_pkt->fname);
360 Jmsg(jcr, M_NOTSAVED, 0, _(" Archive file not saved: %s\n"), ff_pkt->fname);
364 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not open directory %s: ERR=%s\n"),
365 ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
370 Jmsg(jcr, M_NOTSAVED, 0, _(" Unknown file type %d; not saved: %s\n"),
371 ff_pkt->type, ff_pkt->fname);
376 Dmsg1(130, "bfiled: sending %s to stored\n", ff_pkt->fname);
378 /* Digests and encryption are only useful if there's file data */
381 * Setup for digest handling. If this fails, the digest will be set to NULL
382 * and not used. Note, the digest (file hash) can be any one of the four
385 * The signing digest is a single algorithm depending on
386 * whether or not we have SHA2.
387 * ****FIXME**** the signing algoritm should really be
388 * determined a different way!!!!!! What happens if
389 * sha2 was available during backup but not restore?
391 if (ff_pkt->flags & FO_MD5) {
392 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_MD5);
393 digest_stream = STREAM_MD5_DIGEST;
395 } else if (ff_pkt->flags & FO_SHA1) {
396 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA1);
397 digest_stream = STREAM_SHA1_DIGEST;
399 } else if (ff_pkt->flags & FO_SHA256) {
400 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA256);
401 digest_stream = STREAM_SHA256_DIGEST;
403 } else if (ff_pkt->flags & FO_SHA512) {
404 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA512);
405 digest_stream = STREAM_SHA512_DIGEST;
408 /* Did digest initialization fail? */
409 if (digest_stream != STREAM_NONE && digest == NULL) {
410 Jmsg(jcr, M_WARNING, 0, _("%s digest initialization failed\n"),
411 stream_to_ascii(digest_stream));
415 * Set up signature digest handling. If this fails, the signature digest will be set to
418 // TODO landonf: We should really only calculate the digest once, for both verification and signing.
419 if (jcr->crypto.pki_sign) {
420 signing_digest = crypto_digest_new(jcr, signing_algorithm);
422 /* Full-stop if a failure occurred initializing the signature digest */
423 if (signing_digest == NULL) {
424 Jmsg(jcr, M_NOTSAVED, 0, _("%s signature digest initialization failed\n"),
425 stream_to_ascii(signing_algorithm));
431 /* Enable encryption */
432 if (jcr->crypto.pki_encrypt) {
433 ff_pkt->flags |= FO_ENCRYPT;
437 /* Initialize the file descriptor we use for data and other streams. */
439 if (ff_pkt->flags & FO_PORTABLE) {
440 set_portable_backup(&ff_pkt->bfd); /* disable Win32 BackupRead() */
442 if (ff_pkt->cmd_plugin) {
443 if (!set_cmd_plugin(&ff_pkt->bfd, jcr)) {
446 send_plugin_name(jcr, sd, true); /* signal start of plugin data */
449 /* Send attributes -- must be done after binit() */
450 if (!encode_and_send_attributes(jcr, ff_pkt, data_stream)) {
454 /* Set up the encryption context and send the session data to the SD */
455 if (has_file_data && jcr->crypto.pki_encrypt) {
456 if (!crypto_session_send(jcr, sd)) {
462 * Open any file with data that we intend to save, then save it.
464 * Note, if is_win32_backup, we must open the Directory so that
465 * the BackupRead will save its permissions and ownership streams.
467 if (ff_pkt->type != FT_LNKSAVED && S_ISREG(ff_pkt->statp.st_mode)) {
469 do_read = !is_portable_backup(&ff_pkt->bfd) || ff_pkt->statp.st_size > 0;
471 do_read = ff_pkt->statp.st_size > 0;
473 } else if (ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO ||
474 ff_pkt->type == FT_REPARSE ||
475 (!is_portable_backup(&ff_pkt->bfd) && ff_pkt->type == FT_DIREND)) {
477 } else if (ff_pkt->cmd_plugin) {
483 if (ff_pkt->type == FT_FIFO) {
484 tid = start_thread_timer(jcr, pthread_self(), 60);
488 int noatime = ff_pkt->flags & FO_NOATIME ? O_NOATIME : 0;
489 ff_pkt->bfd.reparse_point = ff_pkt->type == FT_REPARSE;
490 if (bopen(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY | noatime, 0) < 0) {
491 ff_pkt->ff_errno = errno;
493 Jmsg(jcr, M_NOTSAVED, 0, _(" Cannot open %s: ERR=%s.\n"), ff_pkt->fname,
497 stop_thread_timer(tid);
503 stop_thread_timer(tid);
507 stat = send_data(jcr, data_stream, ff_pkt, digest, signing_digest);
509 if (ff_pkt->flags & FO_CHKCHANGES) {
510 has_file_changed(jcr, ff_pkt);
513 bclose(&ff_pkt->bfd);
520 #ifdef HAVE_DARWIN_OS
521 /* Regular files can have resource forks and Finder Info */
522 if (ff_pkt->type != FT_LNKSAVED && (S_ISREG(ff_pkt->statp.st_mode) &&
523 ff_pkt->flags & FO_HFSPLUS)) {
524 if (ff_pkt->hfsinfo.rsrclength > 0) {
527 if (!bopen_rsrc(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0) < 0) {
528 ff_pkt->ff_errno = errno;
530 Jmsg(jcr, M_NOTSAVED, -1, _(" Cannot open resource fork for %s: ERR=%s.\n"),
531 ff_pkt->fname, be.bstrerror());
533 if (is_bopen(&ff_pkt->bfd)) {
534 bclose(&ff_pkt->bfd);
538 flags = ff_pkt->flags;
539 ff_pkt->flags &= ~(FO_GZIP|FO_SPARSE);
540 if (flags & FO_ENCRYPT) {
541 rsrc_stream = STREAM_ENCRYPTED_MACOS_FORK_DATA;
543 rsrc_stream = STREAM_MACOS_FORK_DATA;
545 stat = send_data(jcr, rsrc_stream, ff_pkt, digest, signing_digest);
546 ff_pkt->flags = flags;
547 bclose(&ff_pkt->bfd);
553 Dmsg1(300, "Saving Finder Info for \"%s\"\n", ff_pkt->fname);
554 sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_HFSPLUS_ATTRIBUTES);
555 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
556 memcpy(sd->msg, ff_pkt->hfsinfo.fndrinfo, 32);
559 crypto_digest_update(digest, (uint8_t *)sd->msg, sd->msglen);
561 if (signing_digest) {
562 crypto_digest_update(signing_digest, (uint8_t *)sd->msg, sd->msglen);
565 sd->signal(BNET_EOD);
569 if (ff_pkt->flags & FO_ACL) {
570 /* Read access ACLs for files, dirs and links */
571 if (!read_and_send_acl(jcr, BACL_TYPE_ACCESS, STREAM_UNIX_ACCESS_ACL)) {
574 /* Directories can have default ACLs too */
575 if (ff_pkt->type == FT_DIREND && (BACL_CAP & BACL_CAP_DEFAULTS_DIR)) {
576 if (!read_and_send_acl(jcr, BACL_TYPE_DEFAULT, STREAM_UNIX_DEFAULT_ACL)) {
582 /* Terminate the signing digest and send it to the Storage daemon */
583 if (signing_digest) {
586 if ((sig = crypto_sign_new(jcr)) == NULL) {
587 Jmsg(jcr, M_FATAL, 0, _("Failed to allocate memory for crypto signature.\n"));
591 if (!crypto_sign_add_signer(sig, signing_digest, jcr->crypto.pki_keypair)) {
592 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
596 /* Get signature size */
597 if (!crypto_sign_encode(sig, NULL, &size)) {
598 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
602 /* Grow the bsock buffer to fit our message if necessary */
603 if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
604 sd->msg = realloc_pool_memory(sd->msg, size);
607 /* Send our header */
608 sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_SIGNED_DIGEST);
609 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
611 /* Encode signature data */
612 if (!crypto_sign_encode(sig, (uint8_t *)sd->msg, &size)) {
613 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
619 sd->signal(BNET_EOD); /* end of checksum */
622 /* Terminate any digest and send it to Storage daemon */
626 sd->fsend("%ld %d 0", jcr->JobFiles, digest_stream);
627 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
629 size = CRYPTO_DIGEST_MAX_SIZE;
631 /* Grow the bsock buffer to fit our message if necessary */
632 if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
633 sd->msg = realloc_pool_memory(sd->msg, size);
636 if (!crypto_digest_finalize(digest, (uint8_t *)sd->msg, &size)) {
637 Jmsg(jcr, M_FATAL, 0, _("An error occurred finalizing signing the stream.\n"));
643 sd->signal(BNET_EOD); /* end of checksum */
645 if (ff_pkt->cmd_plugin) {
646 send_plugin_name(jcr, sd, false); /* signal end of plugin data */
650 rtnstat = 1; /* good return */
654 crypto_digest_free(digest);
656 if (signing_digest) {
657 crypto_digest_free(signing_digest);
660 crypto_sign_free(sig);
666 * Send data read from an already open file descriptor.
668 * We return 1 on sucess and 0 on errors.
671 * We use ff_pkt->statp.st_size when FO_SPARSE to know when to stop
673 * Currently this is not a problem as the only other stream, resource forks,
674 * are not handled as sparse files.
676 int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest,
677 DIGEST *signing_digest)
679 BSOCK *sd = jcr->store_bsock;
680 uint64_t fileAddr = 0; /* file address */
682 int32_t rsize = jcr->buf_size; /* read buffer size */
684 CIPHER_CONTEXT *cipher_ctx = NULL; /* Quell bogus uninitialized warnings */
685 const uint8_t *cipher_input;
686 uint32_t cipher_input_len;
687 uint32_t cipher_block_size;
688 uint32_t encrypted_len;
689 #ifdef FD_NO_SEND_TEST
694 rbuf = sd->msg; /* read buffer */
695 wbuf = sd->msg; /* write buffer */
696 cipher_input = (uint8_t *)rbuf; /* encrypt uncompressed data */
698 Dmsg1(300, "Saving data, type=%d\n", ff_pkt->type);
701 uLong compress_len = 0;
702 uLong max_compress_len = 0;
703 const Bytef *cbuf = NULL;
706 if (ff_pkt->flags & FO_GZIP) {
707 if (ff_pkt->flags & FO_SPARSE) {
708 cbuf = (Bytef *)jcr->compress_buf + SPARSE_FADDR_SIZE;
709 max_compress_len = jcr->compress_buf_size - SPARSE_FADDR_SIZE;
711 cbuf = (Bytef *)jcr->compress_buf;
712 max_compress_len = jcr->compress_buf_size; /* set max length */
714 wbuf = jcr->compress_buf; /* compressed output here */
715 cipher_input = (uint8_t *)jcr->compress_buf; /* encrypt compressed data */
718 * Only change zlib parameters if there is no pending operation.
719 * This should never happen as deflatereset is called after each
723 if (((z_stream*)jcr->pZLIB_compress_workset)->total_in == 0) {
724 /* set gzip compression level - must be done per file */
725 if ((zstat=deflateParams((z_stream*)jcr->pZLIB_compress_workset,
726 ff_pkt->GZIP_level, Z_DEFAULT_STRATEGY)) != Z_OK) {
727 Jmsg(jcr, M_FATAL, 0, _("Compression deflateParams error: %d\n"), zstat);
728 set_jcr_job_status(jcr, JS_ErrorTerminated);
734 const uint32_t max_compress_len = 0;
737 if (ff_pkt->flags & FO_ENCRYPT) {
738 if (ff_pkt->flags & FO_SPARSE) {
739 Jmsg0(jcr, M_FATAL, 0, _("Encrypting sparse data not supported.\n"));
742 /* Allocate the cipher context */
743 if ((cipher_ctx = crypto_cipher_new(jcr->crypto.pki_session, true,
744 &cipher_block_size)) == NULL) {
745 /* Shouldn't happen! */
746 Jmsg0(jcr, M_FATAL, 0, _("Failed to initialize encryption context.\n"));
751 * Grow the crypto buffer, if necessary.
752 * crypto_cipher_update() will buffer up to (cipher_block_size - 1).
753 * We grow crypto_buf to the maximum number of blocks that
754 * could be returned for the given read buffer size.
755 * (Using the larger of either rsize or max_compress_len)
757 jcr->crypto.crypto_buf = check_pool_memory_size(jcr->crypto.crypto_buf,
758 (MAX(rsize + (int)sizeof(uint32_t), (int32_t)max_compress_len) +
759 cipher_block_size - 1) / cipher_block_size * cipher_block_size);
761 wbuf = jcr->crypto.crypto_buf; /* Encrypted, possibly compressed output here. */
765 * Send Data header to Storage daemon
766 * <file-index> <stream> <info>
768 if (!sd->fsend("%ld %d 0", jcr->JobFiles, stream)) {
769 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
773 Dmsg1(300, ">stored: datahdr %s\n", sd->msg);
776 * Make space at beginning of buffer for fileAddr because this
777 * same buffer will be used for writing if compression is off.
779 if (ff_pkt->flags & FO_SPARSE) {
780 rbuf += SPARSE_FADDR_SIZE;
781 rsize -= SPARSE_FADDR_SIZE;
782 #ifdef HAVE_FREEBSD_OS
784 * To read FreeBSD partitions, the read size must be
787 rsize = (rsize/512) * 512;
791 /* a RAW device read on win32 only works if the buffer is a multiple of 512 */
793 if (S_ISBLK(ff_pkt->statp.st_mode))
794 rsize = (rsize/512) * 512;
800 while ((sd->msglen=(uint32_t)bread(&ff_pkt->bfd, rbuf, rsize)) > 0) {
802 /* Check for sparse blocks */
803 if (ff_pkt->flags & FO_SPARSE) {
805 bool haveBlock = true;
806 if (sd->msglen == rsize &&
807 fileAddr+sd->msglen < (uint64_t)ff_pkt->statp.st_size ||
808 ((ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO) &&
809 (uint64_t)ff_pkt->statp.st_size == 0)) {
810 haveBlock = !is_buf_zero(rbuf, rsize);
813 ser_begin(wbuf, SPARSE_FADDR_SIZE);
814 ser_uint64(fileAddr); /* store fileAddr in begin of buffer */
816 fileAddr += sd->msglen; /* update file address */
818 continue; /* skip block of zeros */
822 jcr->ReadBytes += sd->msglen; /* count bytes read */
824 /* Uncompressed cipher input length */
825 cipher_input_len = sd->msglen;
827 /* Update checksum if requested */
829 crypto_digest_update(digest, (uint8_t *)rbuf, sd->msglen);
832 /* Update signing digest if requested */
833 if (signing_digest) {
834 crypto_digest_update(signing_digest, (uint8_t *)rbuf, sd->msglen);
838 /* Do compression if turned on */
839 if (ff_pkt->flags & FO_GZIP && jcr->pZLIB_compress_workset) {
840 Dmsg3(400, "cbuf=0x%x rbuf=0x%x len=%u\n", cbuf, rbuf, sd->msglen);
842 ((z_stream*)jcr->pZLIB_compress_workset)->next_in = (Bytef *)rbuf;
843 ((z_stream*)jcr->pZLIB_compress_workset)->avail_in = sd->msglen;
844 ((z_stream*)jcr->pZLIB_compress_workset)->next_out = (Bytef *)cbuf;
845 ((z_stream*)jcr->pZLIB_compress_workset)->avail_out = max_compress_len;
847 if ((zstat=deflate((z_stream*)jcr->pZLIB_compress_workset, Z_FINISH)) != Z_STREAM_END) {
848 Jmsg(jcr, M_FATAL, 0, _("Compression deflate error: %d\n"), zstat);
849 set_jcr_job_status(jcr, JS_ErrorTerminated);
852 compress_len = ((z_stream*)jcr->pZLIB_compress_workset)->total_out;
853 /* reset zlib stream to be able to begin from scratch again */
854 if ((zstat=deflateReset((z_stream*)jcr->pZLIB_compress_workset)) != Z_OK) {
855 Jmsg(jcr, M_FATAL, 0, _("Compression deflateReset error: %d\n"), zstat);
856 set_jcr_job_status(jcr, JS_ErrorTerminated);
860 Dmsg2(400, "compressed len=%d uncompressed len=%d\n", compress_len,
863 sd->msglen = compress_len; /* set compressed length */
864 cipher_input_len = compress_len;
868 * Note, here we prepend the current record length to the beginning
869 * of the encrypted data. This is because both sparse and compression
870 * restore handling want records returned to them with exactly the
871 * same number of bytes that were processed in the backup handling.
872 * That is, both are block filters rather than a stream. When doing
873 * compression, the compression routines may buffer data, so that for
874 * any one record compressed, when it is decompressed the same size
875 * will not be obtained. Of course, the buffered data eventually comes
876 * out in subsequent crypto_cipher_update() calls or at least
877 * when crypto_cipher_finalize() is called. Unfortunately, this
878 * "feature" of encryption enormously complicates the restore code.
880 if (ff_pkt->flags & FO_ENCRYPT) {
881 uint32_t initial_len = 0;
884 if (ff_pkt->flags & FO_SPARSE) {
885 cipher_input_len += SPARSE_FADDR_SIZE;
888 /* Encrypt the length of the input block */
889 uint8_t packet_len[sizeof(uint32_t)];
891 ser_begin(packet_len, sizeof(uint32_t));
892 ser_uint32(cipher_input_len); /* store data len in begin of buffer */
893 Dmsg1(20, "Encrypt len=%d\n", cipher_input_len);
895 if (!crypto_cipher_update(cipher_ctx, packet_len, sizeof(packet_len),
896 (uint8_t *)jcr->crypto.crypto_buf, &initial_len)) {
897 /* Encryption failed. Shouldn't happen. */
898 Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
902 /* Encrypt the input block */
903 if (crypto_cipher_update(cipher_ctx, cipher_input, cipher_input_len,
904 (uint8_t *)&jcr->crypto.crypto_buf[initial_len], &encrypted_len)) {
905 if ((initial_len + encrypted_len) == 0) {
906 /* No full block of data available, read more data */
909 Dmsg2(400, "encrypted len=%d unencrypted len=%d\n", encrypted_len,
911 sd->msglen = initial_len + encrypted_len; /* set encrypted length */
913 /* Encryption failed. Shouldn't happen. */
914 Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
919 /* Send the buffer to the Storage daemon */
920 if (ff_pkt->flags & FO_SPARSE) {
921 sd->msglen += SPARSE_FADDR_SIZE; /* include fileAddr in size */
923 sd->msg = wbuf; /* set correct write buffer */
925 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
929 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
931 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
932 sd->msg = msgsave; /* restore read buffer */
934 } /* end while read file data */
936 if (sd->msglen < 0) { /* error */
938 Jmsg(jcr, M_ERROR, 0, _("Read error on file %s. ERR=%s\n"),
939 ff_pkt->fname, be.bstrerror(ff_pkt->bfd.berrno));
940 if (jcr->Errors++ > 1000) { /* insanity check */
941 Jmsg(jcr, M_FATAL, 0, _("Too many errors.\n"));
943 } else if (ff_pkt->flags & FO_ENCRYPT) {
945 * For encryption, we must call finalize to push out any
948 if (!crypto_cipher_finalize(cipher_ctx, (uint8_t *)jcr->crypto.crypto_buf,
950 /* Padding failed. Shouldn't happen. */
951 Jmsg(jcr, M_FATAL, 0, _("Encryption padding error\n"));
955 /* Note, on SSL pre-0.9.7, there is always some output */
956 if (encrypted_len > 0) {
957 sd->msglen = encrypted_len; /* set encrypted length */
958 sd->msg = jcr->crypto.crypto_buf; /* set correct write buffer */
960 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
964 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
965 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
966 sd->msg = msgsave; /* restore bnet buffer */
970 if (!sd->signal(BNET_EOD)) { /* indicate end of file data */
971 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
976 /* Free the cipher context */
978 crypto_cipher_free(cipher_ctx);
983 /* Free the cipher context */
985 crypto_cipher_free(cipher_ctx);
988 sd->msg = msgsave; /* restore bnet buffer */
994 * Read and send an ACL for the last encountered file.
996 static bool read_and_send_acl(JCR *jcr, int acltype, int stream)
999 BSOCK *sd = jcr->store_bsock;
1002 #ifdef FD_NO_SEND_TEST
1006 len = bacl_get(jcr, acltype);
1008 Jmsg1(jcr, M_WARNING, 0, _("Error reading ACL of %s\n"), jcr->last_fname);
1012 return true; /* no ACL */
1016 if (!sd->fsend("%ld %d 0", jcr->JobFiles, stream)) {
1017 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1022 /* Send the buffer to the storage deamon */
1023 Dmsg2(400, "Backing up ACL type 0x%2x <%s>\n", acltype, jcr->acl_text);
1025 sd->msg = jcr->acl_text;
1026 sd->msglen = len + 1;
1030 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1035 jcr->JobBytes += sd->msglen;
1037 if (!sd->signal(BNET_EOD)) {
1038 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1043 Dmsg1(200, "ACL of file: %s successfully backed up!\n", jcr->last_fname);
1048 static bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream)
1050 BSOCK *sd = jcr->store_bsock;
1051 char attribs[MAXSTRING];
1052 char attribsEx[MAXSTRING];
1055 #ifdef FD_NO_SEND_TEST
1059 Dmsg1(300, "encode_and_send_attrs fname=%s\n", ff_pkt->fname);
1060 /* Find what data stream we will use, then encode the attributes */
1061 if ((data_stream = select_data_stream(ff_pkt)) == STREAM_NONE) {
1062 /* This should not happen */
1063 Jmsg0(jcr, M_FATAL, 0, _("Invalid file flags, no supported data stream type.\n"));
1066 encode_stat(attribs, ff_pkt, data_stream);
1068 /* Now possibly extend the attributes */
1069 attr_stream = encode_attribsEx(jcr, attribsEx, ff_pkt);
1071 Dmsg3(300, "File %s\nattribs=%s\nattribsEx=%s\n", ff_pkt->fname, attribs, attribsEx);
1074 jcr->JobFiles++; /* increment number of files sent */
1075 ff_pkt->FileIndex = jcr->JobFiles; /* return FileIndex */
1076 pm_strcpy(jcr->last_fname, ff_pkt->fname);
1080 * Send Attributes header to Storage daemon
1081 * <file-index> <stream> <info>
1083 if (!sd->fsend("%ld %d 0", jcr->JobFiles, attr_stream)) {
1084 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1088 Dmsg1(300, ">stored: attrhdr %s\n", sd->msg);
1091 * Send file attributes to Storage daemon
1094 * Filename (full path)
1095 * Encoded attributes
1096 * Link name (if type==FT_LNK or FT_LNKSAVED)
1097 * Encoded extended-attributes (for Win32)
1099 * For a directory, link is the same as fname, but with trailing
1100 * slash. For a linked file, link is the link.
1103 if (ff_pkt->type == FT_LNK || ff_pkt->type == FT_LNKSAVED) {
1104 Dmsg2(300, "Link %s to %s\n", ff_pkt->fname, ff_pkt->link);
1105 stat = sd->fsend("%ld %d %s%c%s%c%s%c%s%c", jcr->JobFiles,
1106 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, ff_pkt->link, 0,
1108 } else if (ff_pkt->type == FT_DIREND || ff_pkt->type == FT_REPARSE) {
1109 /* Here link is the canonical filename (i.e. with trailing slash) */
1110 stat = sd->fsend("%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
1111 ff_pkt->type, ff_pkt->link, 0, attribs, 0, 0, attribsEx, 0);
1113 stat = sd->fsend("%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
1114 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, 0, attribsEx, 0);
1116 unstrip_path(ff_pkt);
1118 Dmsg2(300, ">stored: attr len=%d: %s\n", sd->msglen, sd->msg);
1120 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1124 sd->signal(BNET_EOD); /* indicate end of attributes data */
1129 * Do in place strip of path
1131 static bool do_strip(int count, char *in)
1137 /* Copy to first path separator -- Win32 might have c: ... */
1138 while (*in && !IsPathSeparator(*in)) {
1142 numsep++; /* one separator seen */
1143 for (stripped=0; stripped<count && *in; stripped++) {
1144 while (*in && !IsPathSeparator(*in)) {
1145 in++; /* skip chars */
1148 numsep++; /* count separators seen */
1149 in++; /* skip separator */
1153 while (*in) { /* copy to end */
1154 if (IsPathSeparator(*in)) {
1160 Dmsg4(500, "stripped=%d count=%d numsep=%d sep>count=%d\n",
1161 stripped, count, numsep, numsep>count);
1162 return stripped==count && numsep>count;
1166 * If requested strip leading components of the path
1168 static void strip_path(FF_PKT *ff_pkt)
1170 if (!(ff_pkt->flags & FO_STRIPPATH) || ff_pkt->strip_path <= 0) {
1171 Dmsg1(200, "No strip for %s\n", ff_pkt->fname);
1174 if (!ff_pkt->fname_save) {
1175 ff_pkt->fname_save = get_pool_memory(PM_FNAME);
1176 ff_pkt->link_save = get_pool_memory(PM_FNAME);
1178 pm_strcpy(ff_pkt->fname_save, ff_pkt->fname);
1181 * Strip path. If it doesn't succeed put it back. If
1182 * it does, and there is a different link string,
1183 * attempt to strip the link. If it fails, back them
1185 * Do not strip symlinks.
1186 * I.e. if either stripping fails don't strip anything.
1188 if (do_strip(ff_pkt->strip_path, ff_pkt->fname)) {
1189 /* Strip links but not symlinks */
1190 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1191 pm_strcpy(ff_pkt->link_save, ff_pkt->link);
1192 if (!do_strip(ff_pkt->strip_path, ff_pkt->link)) {
1193 pm_strcpy(ff_pkt->link, ff_pkt->link_save);
1194 pm_strcpy(ff_pkt->fname, ff_pkt->fname_save);
1198 pm_strcpy(ff_pkt->fname, ff_pkt->fname_save);
1200 Dmsg2(200, "fname=%s stripped=%s\n", ff_pkt->fname_save, ff_pkt->fname);
1203 static void unstrip_path(FF_PKT *ff_pkt)
1205 if (!(ff_pkt->flags & FO_STRIPPATH) || ff_pkt->strip_path <= 0) {
1208 pm_strcpy(ff_pkt->fname, ff_pkt->fname_save);
1209 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1210 pm_strcpy(ff_pkt->link, ff_pkt->link_save);