2 Bacula® - The Network Backup Solution
4 Copyright (C) 2000-2009 Free Software Foundation Europe e.V.
6 The main author of Bacula is Kern Sibbald, with contributions from
7 many others, a complete list can be found in the file AUTHORS.
8 This program is Free Software; you can redistribute it and/or
9 modify it under the terms of version two of the GNU General Public
10 License as published by the Free Software Foundation and included
13 This program is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
23 Bacula® is a registered trademark of Kern Sibbald.
24 The licensor of Bacula is the Free Software Foundation Europe
25 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
26 Switzerland, email:ftf@fsfeurope.org.
29 * Bacula File Daemon backup.c send file attributes and data
30 * to the Storage daemon.
32 * Kern Sibbald, March MM
42 const bool have_acl = true;
44 const bool have_acl = false;
47 #if defined(HAVE_XATTR)
48 const bool have_xattr = true;
50 const bool have_xattr = false;
53 /* Forward referenced functions */
54 int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level);
55 static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, DIGEST *signature_digest);
56 bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream);
57 static bool crypto_session_start(JCR *jcr);
58 static void crypto_session_end(JCR *jcr);
59 static bool crypto_session_send(JCR *jcr, BSOCK *sd);
62 * Find all the requested files and send them
63 * to the Storage daemon.
65 * Note, we normally carry on a one-way
66 * conversation from this point on with the SD, simply blasting
67 * data to him. To properly know what is going on, we
68 * also run a "heartbeat" monitor which reads the socket and
69 * reacts accordingly (at the moment it has nothing to do
70 * except echo the heartbeat to the Director).
73 bool blast_data_to_storage_daemon(JCR *jcr, char *addr)
77 // TODO landonf: Allow user to specify encryption algorithm
79 sd = jcr->store_bsock;
81 set_jcr_job_status(jcr, JS_Running);
83 Dmsg1(300, "bfiled: opened data connection %d to stored\n", sd->m_fd);
86 CLIENT *client = (CLIENT *)GetNextRes(R_CLIENT, NULL);
90 buf_size = client->max_network_buffer_size;
92 buf_size = 0; /* use default */
94 if (!sd->set_buffer_size(buf_size, BNET_SETBUF_WRITE)) {
95 set_jcr_job_status(jcr, JS_ErrorTerminated);
96 Jmsg(jcr, M_FATAL, 0, _("Cannot set buffer size FD->SD.\n"));
100 jcr->buf_size = sd->msglen;
101 /* Adjust for compression so that output buffer is
102 * 12 bytes + 0.1% larger than input buffer plus 18 bytes.
103 * This gives a bit extra plus room for the sparse addr if any.
104 * Note, we adjust the read size to be smaller so that the
105 * same output buffer can be used without growing it.
107 * The zlib compression workset is initialized here to minimize
108 * the "per file" load. The jcr member is only set, if the init
111 jcr->compress_buf_size = jcr->buf_size + ((jcr->buf_size+999) / 1000) + 30;
112 jcr->compress_buf = get_memory(jcr->compress_buf_size);
115 z_stream *pZlibStream = (z_stream*)malloc(sizeof(z_stream));
117 pZlibStream->zalloc = Z_NULL;
118 pZlibStream->zfree = Z_NULL;
119 pZlibStream->opaque = Z_NULL;
120 pZlibStream->state = Z_NULL;
122 if (deflateInit(pZlibStream, Z_DEFAULT_COMPRESSION) == Z_OK) {
123 jcr->pZLIB_compress_workset = pZlibStream;
130 if (!crypto_session_start(jcr)) {
134 set_find_options((FF_PKT *)jcr->ff, jcr->incremental, jcr->mtime);
136 /* in accurate mode, we overwrite the find_one check function */
138 set_find_changed_function((FF_PKT *)jcr->ff, accurate_check_file);
141 start_heartbeat_monitor(jcr);
144 jcr->acl_data = get_pool_memory(PM_MESSAGE);
145 jcr->total_acl_errors = 0;
148 jcr->xattr_data = get_pool_memory(PM_MESSAGE);
149 jcr->total_xattr_errors = 0;
152 /* Subroutine save_file() is called for each file */
153 if (!find_files(jcr, (FF_PKT *)jcr->ff, save_file, plugin_save)) {
154 ok = false; /* error */
155 set_jcr_job_status(jcr, JS_ErrorTerminated);
158 if (jcr->total_acl_errors > 0) {
159 Jmsg(jcr, M_ERROR, 0, _("Encountered %ld acl errors while doing backup\n"),
160 jcr->total_acl_errors);
162 if (jcr->total_xattr_errors > 0) {
163 Jmsg(jcr, M_ERROR, 0, _("Encountered %ld xattr errors while doing backup\n"),
164 jcr->total_xattr_errors);
167 accurate_send_deleted_list(jcr); /* send deleted list to SD */
169 stop_heartbeat_monitor(jcr);
171 sd->signal(BNET_EOD); /* end of sending data */
173 if (have_acl && jcr->acl_data) {
174 free_pool_memory(jcr->acl_data);
175 jcr->acl_data = NULL;
177 if (have_xattr && jcr->xattr_data) {
178 free_pool_memory(jcr->xattr_data);
179 jcr->xattr_data = NULL;
185 if (jcr->compress_buf) {
186 free_pool_memory(jcr->compress_buf);
187 jcr->compress_buf = NULL;
189 if (jcr->pZLIB_compress_workset) {
190 /* Free the zlib stream */
192 deflateEnd((z_stream *)jcr->pZLIB_compress_workset);
194 free (jcr->pZLIB_compress_workset);
195 jcr->pZLIB_compress_workset = NULL;
197 crypto_session_end(jcr);
200 Dmsg1(100, "end blast_data ok=%d\n", ok);
204 static bool crypto_session_start(JCR *jcr)
206 crypto_cipher_t cipher = CRYPTO_CIPHER_AES_128_CBC;
209 * Create encryption session data and a cached, DER-encoded session data
210 * structure. We use a single session key for each backup, so we'll encode
211 * the session data only once.
213 if (jcr->crypto.pki_encrypt) {
216 /* Create per-job session encryption context */
217 jcr->crypto.pki_session = crypto_session_new(cipher, jcr->crypto.pki_recipients);
219 /* Get the session data size */
220 if (!crypto_session_encode(jcr->crypto.pki_session, (uint8_t *)0, &size)) {
221 Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n"));
225 /* Allocate buffer */
226 jcr->crypto.pki_session_encoded = get_memory(size);
228 /* Encode session data */
229 if (!crypto_session_encode(jcr->crypto.pki_session, (uint8_t *)jcr->crypto.pki_session_encoded, &size)) {
230 Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n"));
234 /* ... and store the encoded size */
235 jcr->crypto.pki_session_encoded_size = size;
237 /* Allocate the encryption/decryption buffer */
238 jcr->crypto.crypto_buf = get_memory(CRYPTO_CIPHER_MAX_BLOCK_SIZE);
243 static void crypto_session_end(JCR *jcr)
245 if (jcr->crypto.crypto_buf) {
246 free_pool_memory(jcr->crypto.crypto_buf);
247 jcr->crypto.crypto_buf = NULL;
249 if (jcr->crypto.pki_session) {
250 crypto_session_free(jcr->crypto.pki_session);
252 if (jcr->crypto.pki_session_encoded) {
253 free_pool_memory(jcr->crypto.pki_session_encoded);
254 jcr->crypto.pki_session_encoded = NULL;
258 static bool crypto_session_send(JCR *jcr, BSOCK *sd)
262 /* Send our header */
263 Dmsg2(100, "Send hdr fi=%ld stream=%d\n", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA);
264 sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA);
267 sd->msg = jcr->crypto.pki_session_encoded;
268 sd->msglen = jcr->crypto.pki_session_encoded_size;
269 jcr->JobBytes += sd->msglen;
271 Dmsg1(100, "Send data len=%d\n", sd->msglen);
274 sd->signal(BNET_EOD);
280 * Called here by find() for each file included.
281 * This is a callback. The original is find_files() above.
283 * Send the file and its data to the Storage daemon.
287 * -1 to ignore file/directory (not used here)
289 int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level)
291 bool do_read = false;
292 int stat, data_stream;
294 DIGEST *digest = NULL;
295 DIGEST *signing_digest = NULL;
296 int digest_stream = STREAM_NONE;
297 SIGNATURE *sig = NULL;
298 bool has_file_data = false;
299 // TODO landonf: Allow the user to specify the digest algorithm
301 crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA256;
303 crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA1;
305 BSOCK *sd = jcr->store_bsock;
307 if (job_canceled(jcr)) {
311 jcr->num_files_examined++; /* bump total file count */
313 switch (ff_pkt->type) {
314 case FT_LNKSAVED: /* Hard linked, file already saved */
315 Dmsg2(130, "FT_LNKSAVED hard link: %s => %s\n", ff_pkt->fname, ff_pkt->link);
318 Dmsg1(130, "FT_REGE saving: %s\n", ff_pkt->fname);
319 has_file_data = true;
322 Dmsg1(130, "FT_REG saving: %s\n", ff_pkt->fname);
323 has_file_data = true;
326 Dmsg2(130, "FT_LNK saving: %s -> %s\n", ff_pkt->fname, ff_pkt->link);
329 jcr->num_files_examined--; /* correct file count */
330 return 1; /* not used */
332 Jmsg(jcr, M_INFO, 1, _(" Recursion turned off. Will not descend from %s into %s\n"),
333 ff_pkt->top_fname, ff_pkt->fname);
334 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
337 /* Suppress message for /dev filesystems */
338 if (!is_in_fileset(ff_pkt)) {
339 Jmsg(jcr, M_INFO, 1, _(" %s is a different filesystem. Will not descend from %s into %s\n"),
340 ff_pkt->fname, ff_pkt->top_fname, ff_pkt->fname);
342 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
345 Jmsg(jcr, M_INFO, 1, _(" Disallowed filesystem. Will not descend from %s into %s\n"),
346 ff_pkt->top_fname, ff_pkt->fname);
347 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
350 Jmsg(jcr, M_INFO, 1, _(" Disallowed drive type. Will not descend into %s\n"),
355 Dmsg1(130, "FT_DIREND: %s\n", ff_pkt->link);
358 Dmsg1(130, "FT_SPEC saving: %s\n", ff_pkt->fname);
359 if (S_ISSOCK(ff_pkt->statp.st_mode)) {
360 Jmsg(jcr, M_SKIPPED, 1, _(" Socket file skipped: %s\n"), ff_pkt->fname);
365 Dmsg1(130, "FT_RAW saving: %s\n", ff_pkt->fname);
366 has_file_data = true;
369 Dmsg1(130, "FT_FIFO saving: %s\n", ff_pkt->fname);
373 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not access \"%s\": ERR=%s\n"), ff_pkt->fname,
374 be.bstrerror(ff_pkt->ff_errno));
380 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not follow link \"%s\": ERR=%s\n"),
381 ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
387 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not stat \"%s\": ERR=%s\n"), ff_pkt->fname,
388 be.bstrerror(ff_pkt->ff_errno));
394 Jmsg(jcr, M_SKIPPED, 1, _(" Unchanged file skipped: %s\n"), ff_pkt->fname);
397 Jmsg(jcr, M_NOTSAVED, 0, _(" Archive file not saved: %s\n"), ff_pkt->fname);
401 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not open directory \"%s\": ERR=%s\n"),
402 ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
407 Jmsg(jcr, M_NOTSAVED, 0, _(" Unknown file type %d; not saved: %s\n"),
408 ff_pkt->type, ff_pkt->fname);
413 Dmsg1(130, "bfiled: sending %s to stored\n", ff_pkt->fname);
415 /* Digests and encryption are only useful if there's file data */
418 * Setup for digest handling. If this fails, the digest will be set to NULL
419 * and not used. Note, the digest (file hash) can be any one of the four
422 * The signing digest is a single algorithm depending on
423 * whether or not we have SHA2.
424 * ****FIXME**** the signing algoritm should really be
425 * determined a different way!!!!!! What happens if
426 * sha2 was available during backup but not restore?
428 if (ff_pkt->flags & FO_MD5) {
429 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_MD5);
430 digest_stream = STREAM_MD5_DIGEST;
432 } else if (ff_pkt->flags & FO_SHA1) {
433 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA1);
434 digest_stream = STREAM_SHA1_DIGEST;
436 } else if (ff_pkt->flags & FO_SHA256) {
437 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA256);
438 digest_stream = STREAM_SHA256_DIGEST;
440 } else if (ff_pkt->flags & FO_SHA512) {
441 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA512);
442 digest_stream = STREAM_SHA512_DIGEST;
445 /* Did digest initialization fail? */
446 if (digest_stream != STREAM_NONE && digest == NULL) {
447 Jmsg(jcr, M_WARNING, 0, _("%s digest initialization failed\n"),
448 stream_to_ascii(digest_stream));
452 * Set up signature digest handling. If this fails, the signature digest will be set to
455 // TODO landonf: We should really only calculate the digest once, for both verification and signing.
456 if (jcr->crypto.pki_sign) {
457 signing_digest = crypto_digest_new(jcr, signing_algorithm);
459 /* Full-stop if a failure occurred initializing the signature digest */
460 if (signing_digest == NULL) {
461 Jmsg(jcr, M_NOTSAVED, 0, _("%s signature digest initialization failed\n"),
462 stream_to_ascii(signing_algorithm));
468 /* Enable encryption */
469 if (jcr->crypto.pki_encrypt) {
470 ff_pkt->flags |= FO_ENCRYPT;
474 /* Initialize the file descriptor we use for data and other streams. */
476 if (ff_pkt->flags & FO_PORTABLE) {
477 set_portable_backup(&ff_pkt->bfd); /* disable Win32 BackupRead() */
479 if (ff_pkt->cmd_plugin) {
480 if (!set_cmd_plugin(&ff_pkt->bfd, jcr)) {
483 send_plugin_name(jcr, sd, true); /* signal start of plugin data */
486 /* Send attributes -- must be done after binit() */
487 if (!encode_and_send_attributes(jcr, ff_pkt, data_stream)) {
491 /* Set up the encryption context and send the session data to the SD */
492 if (has_file_data && jcr->crypto.pki_encrypt) {
493 if (!crypto_session_send(jcr, sd)) {
499 * Open any file with data that we intend to save, then save it.
501 * Note, if is_win32_backup, we must open the Directory so that
502 * the BackupRead will save its permissions and ownership streams.
504 if (ff_pkt->type != FT_LNKSAVED && S_ISREG(ff_pkt->statp.st_mode)) {
506 do_read = !is_portable_backup(&ff_pkt->bfd) || ff_pkt->statp.st_size > 0;
508 do_read = ff_pkt->statp.st_size > 0;
510 } else if (ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO ||
511 ff_pkt->type == FT_REPARSE ||
512 (!is_portable_backup(&ff_pkt->bfd) && ff_pkt->type == FT_DIREND)) {
515 if (ff_pkt->cmd_plugin) {
519 Dmsg1(400, "do_read=%d\n", do_read);
523 if (ff_pkt->type == FT_FIFO) {
524 tid = start_thread_timer(jcr, pthread_self(), 60);
528 int noatime = ff_pkt->flags & FO_NOATIME ? O_NOATIME : 0;
529 ff_pkt->bfd.reparse_point = ff_pkt->type == FT_REPARSE;
530 if (bopen(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY | noatime, 0) < 0) {
531 ff_pkt->ff_errno = errno;
533 Jmsg(jcr, M_NOTSAVED, 0, _(" Cannot open \"%s\": ERR=%s.\n"), ff_pkt->fname,
537 stop_thread_timer(tid);
543 stop_thread_timer(tid);
547 stat = send_data(jcr, data_stream, ff_pkt, digest, signing_digest);
549 if (ff_pkt->flags & FO_CHKCHANGES) {
550 has_file_changed(jcr, ff_pkt);
553 bclose(&ff_pkt->bfd);
560 #ifdef HAVE_DARWIN_OS
561 /* Regular files can have resource forks and Finder Info */
562 if (ff_pkt->type != FT_LNKSAVED && (S_ISREG(ff_pkt->statp.st_mode) &&
563 ff_pkt->flags & FO_HFSPLUS)) {
564 if (ff_pkt->hfsinfo.rsrclength > 0) {
567 if (!bopen_rsrc(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0) < 0) {
568 ff_pkt->ff_errno = errno;
570 Jmsg(jcr, M_NOTSAVED, -1, _(" Cannot open resource fork for \"%s\": ERR=%s.\n"),
571 ff_pkt->fname, be.bstrerror());
573 if (is_bopen(&ff_pkt->bfd)) {
574 bclose(&ff_pkt->bfd);
578 flags = ff_pkt->flags;
579 ff_pkt->flags &= ~(FO_GZIP|FO_SPARSE);
580 if (flags & FO_ENCRYPT) {
581 rsrc_stream = STREAM_ENCRYPTED_MACOS_FORK_DATA;
583 rsrc_stream = STREAM_MACOS_FORK_DATA;
585 stat = send_data(jcr, rsrc_stream, ff_pkt, digest, signing_digest);
586 ff_pkt->flags = flags;
587 bclose(&ff_pkt->bfd);
593 Dmsg1(300, "Saving Finder Info for \"%s\"\n", ff_pkt->fname);
594 sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_HFSPLUS_ATTRIBUTES);
595 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
596 pm_memcpy(sd->msg, ff_pkt->hfsinfo.fndrinfo, 32);
599 crypto_digest_update(digest, (uint8_t *)sd->msg, sd->msglen);
601 if (signing_digest) {
602 crypto_digest_update(signing_digest, (uint8_t *)sd->msg, sd->msglen);
605 sd->signal(BNET_EOD);
610 * Save ACLs when requested and available for anything not being a symlink and not being a plugin.
613 if (ff_pkt->flags & FO_ACL && ff_pkt->type != FT_LNK && !ff_pkt->cmd_plugin) {
614 switch (build_acl_streams(jcr, ff_pkt)) {
615 case bacl_exit_fatal:
617 case bacl_exit_error:
619 * Non-fatal errors, count them and when the number is under ACL_REPORT_ERR_MAX_PER_JOB
620 * print the error message set by the lower level routine in jcr->errmsg.
622 if (jcr->total_acl_errors < ACL_REPORT_ERR_MAX_PER_JOB) {
623 Jmsg(jcr, M_ERROR, 0, "%s", jcr->errmsg);
625 jcr->total_acl_errors++;
634 * Save Extended Attributes when requested and available for all files not being a plugin.
637 if (ff_pkt->flags & FO_XATTR && !ff_pkt->cmd_plugin) {
638 switch (build_xattr_streams(jcr, ff_pkt)) {
639 case bxattr_exit_fatal:
641 case bxattr_exit_error:
643 * Non-fatal errors, count them and when the number is under XATTR_REPORT_ERR_MAX_PER_JOB
644 * print the error message set by the lower level routine in jcr->errmsg.
646 if (jcr->total_xattr_errors < XATTR_REPORT_ERR_MAX_PER_JOB) {
647 Jmsg(jcr, M_ERROR, 0, "%s", jcr->errmsg);
649 jcr->total_xattr_errors++;
657 /* Terminate the signing digest and send it to the Storage daemon */
658 if (signing_digest) {
661 if ((sig = crypto_sign_new(jcr)) == NULL) {
662 Jmsg(jcr, M_FATAL, 0, _("Failed to allocate memory for crypto signature.\n"));
666 if (!crypto_sign_add_signer(sig, signing_digest, jcr->crypto.pki_keypair)) {
667 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
671 /* Get signature size */
672 if (!crypto_sign_encode(sig, NULL, &size)) {
673 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
677 /* Grow the bsock buffer to fit our message if necessary */
678 if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
679 sd->msg = realloc_pool_memory(sd->msg, size);
682 /* Send our header */
683 sd->fsend("%ld %ld 0", jcr->JobFiles, STREAM_SIGNED_DIGEST);
684 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
686 /* Encode signature data */
687 if (!crypto_sign_encode(sig, (uint8_t *)sd->msg, &size)) {
688 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
694 sd->signal(BNET_EOD); /* end of checksum */
697 /* Terminate any digest and send it to Storage daemon */
701 sd->fsend("%ld %d 0", jcr->JobFiles, digest_stream);
702 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
704 size = CRYPTO_DIGEST_MAX_SIZE;
706 /* Grow the bsock buffer to fit our message if necessary */
707 if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
708 sd->msg = realloc_pool_memory(sd->msg, size);
711 if (!crypto_digest_finalize(digest, (uint8_t *)sd->msg, &size)) {
712 Jmsg(jcr, M_FATAL, 0, _("An error occurred finalizing signing the stream.\n"));
718 sd->signal(BNET_EOD); /* end of checksum */
720 if (ff_pkt->cmd_plugin) {
721 send_plugin_name(jcr, sd, false); /* signal end of plugin data */
725 rtnstat = 1; /* good return */
729 crypto_digest_free(digest);
731 if (signing_digest) {
732 crypto_digest_free(signing_digest);
735 crypto_sign_free(sig);
741 * Send data read from an already open file descriptor.
743 * We return 1 on sucess and 0 on errors.
746 * We use ff_pkt->statp.st_size when FO_SPARSE to know when to stop
748 * Currently this is not a problem as the only other stream, resource forks,
749 * are not handled as sparse files.
751 static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest,
752 DIGEST *signing_digest)
754 BSOCK *sd = jcr->store_bsock;
755 uint64_t fileAddr = 0; /* file address */
757 int32_t rsize = jcr->buf_size; /* read buffer size */
759 CIPHER_CONTEXT *cipher_ctx = NULL; /* Quell bogus uninitialized warnings */
760 const uint8_t *cipher_input;
761 uint32_t cipher_input_len;
762 uint32_t cipher_block_size;
763 uint32_t encrypted_len;
764 #ifdef FD_NO_SEND_TEST
769 rbuf = sd->msg; /* read buffer */
770 wbuf = sd->msg; /* write buffer */
771 cipher_input = (uint8_t *)rbuf; /* encrypt uncompressed data */
773 Dmsg1(300, "Saving data, type=%d\n", ff_pkt->type);
776 uLong compress_len = 0;
777 uLong max_compress_len = 0;
778 const Bytef *cbuf = NULL;
781 if (ff_pkt->flags & FO_GZIP) {
782 if (ff_pkt->flags & FO_SPARSE) {
783 cbuf = (Bytef *)jcr->compress_buf + SPARSE_FADDR_SIZE;
784 max_compress_len = jcr->compress_buf_size - SPARSE_FADDR_SIZE;
786 cbuf = (Bytef *)jcr->compress_buf;
787 max_compress_len = jcr->compress_buf_size; /* set max length */
789 wbuf = jcr->compress_buf; /* compressed output here */
790 cipher_input = (uint8_t *)jcr->compress_buf; /* encrypt compressed data */
793 * Only change zlib parameters if there is no pending operation.
794 * This should never happen as deflatereset is called after each
798 if (((z_stream*)jcr->pZLIB_compress_workset)->total_in == 0) {
799 /* set gzip compression level - must be done per file */
800 if ((zstat=deflateParams((z_stream*)jcr->pZLIB_compress_workset,
801 ff_pkt->GZIP_level, Z_DEFAULT_STRATEGY)) != Z_OK) {
802 Jmsg(jcr, M_FATAL, 0, _("Compression deflateParams error: %d\n"), zstat);
803 set_jcr_job_status(jcr, JS_ErrorTerminated);
809 const uint32_t max_compress_len = 0;
812 if (ff_pkt->flags & FO_ENCRYPT) {
813 if (ff_pkt->flags & FO_SPARSE) {
814 Jmsg0(jcr, M_FATAL, 0, _("Encrypting sparse data not supported.\n"));
817 /* Allocate the cipher context */
818 if ((cipher_ctx = crypto_cipher_new(jcr->crypto.pki_session, true,
819 &cipher_block_size)) == NULL) {
820 /* Shouldn't happen! */
821 Jmsg0(jcr, M_FATAL, 0, _("Failed to initialize encryption context.\n"));
826 * Grow the crypto buffer, if necessary.
827 * crypto_cipher_update() will buffer up to (cipher_block_size - 1).
828 * We grow crypto_buf to the maximum number of blocks that
829 * could be returned for the given read buffer size.
830 * (Using the larger of either rsize or max_compress_len)
832 jcr->crypto.crypto_buf = check_pool_memory_size(jcr->crypto.crypto_buf,
833 (MAX(rsize + (int)sizeof(uint32_t), (int32_t)max_compress_len) +
834 cipher_block_size - 1) / cipher_block_size * cipher_block_size);
836 wbuf = jcr->crypto.crypto_buf; /* Encrypted, possibly compressed output here. */
840 * Send Data header to Storage daemon
841 * <file-index> <stream> <info>
843 if (!sd->fsend("%ld %d 0", jcr->JobFiles, stream)) {
844 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
848 Dmsg1(300, ">stored: datahdr %s\n", sd->msg);
851 * Make space at beginning of buffer for fileAddr because this
852 * same buffer will be used for writing if compression is off.
854 if (ff_pkt->flags & FO_SPARSE) {
855 rbuf += SPARSE_FADDR_SIZE;
856 rsize -= SPARSE_FADDR_SIZE;
857 #ifdef HAVE_FREEBSD_OS
859 * To read FreeBSD partitions, the read size must be
862 rsize = (rsize/512) * 512;
866 /* a RAW device read on win32 only works if the buffer is a multiple of 512 */
868 if (S_ISBLK(ff_pkt->statp.st_mode))
869 rsize = (rsize/512) * 512;
875 while ((sd->msglen=(uint32_t)bread(&ff_pkt->bfd, rbuf, rsize)) > 0) {
877 /* Check for sparse blocks */
878 if (ff_pkt->flags & FO_SPARSE) {
880 bool allZeros = false;
881 if ((sd->msglen == rsize &&
882 fileAddr+sd->msglen < (uint64_t)ff_pkt->statp.st_size) ||
883 ((ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO) &&
884 (uint64_t)ff_pkt->statp.st_size == 0)) {
885 allZeros = is_buf_zero(rbuf, rsize);
888 /* Put file address as first data in buffer */
889 ser_begin(wbuf, SPARSE_FADDR_SIZE);
890 ser_uint64(fileAddr); /* store fileAddr in begin of buffer */
892 fileAddr += sd->msglen; /* update file address */
893 /* Skip block of all zeros */
895 continue; /* skip block of zeros */
899 jcr->ReadBytes += sd->msglen; /* count bytes read */
901 /* Uncompressed cipher input length */
902 cipher_input_len = sd->msglen;
904 /* Update checksum if requested */
906 crypto_digest_update(digest, (uint8_t *)rbuf, sd->msglen);
909 /* Update signing digest if requested */
910 if (signing_digest) {
911 crypto_digest_update(signing_digest, (uint8_t *)rbuf, sd->msglen);
915 /* Do compression if turned on */
916 if (ff_pkt->flags & FO_GZIP && jcr->pZLIB_compress_workset) {
917 Dmsg3(400, "cbuf=0x%x rbuf=0x%x len=%u\n", cbuf, rbuf, sd->msglen);
919 ((z_stream*)jcr->pZLIB_compress_workset)->next_in = (Bytef *)rbuf;
920 ((z_stream*)jcr->pZLIB_compress_workset)->avail_in = sd->msglen;
921 ((z_stream*)jcr->pZLIB_compress_workset)->next_out = (Bytef *)cbuf;
922 ((z_stream*)jcr->pZLIB_compress_workset)->avail_out = max_compress_len;
924 if ((zstat=deflate((z_stream*)jcr->pZLIB_compress_workset, Z_FINISH)) != Z_STREAM_END) {
925 Jmsg(jcr, M_FATAL, 0, _("Compression deflate error: %d\n"), zstat);
926 set_jcr_job_status(jcr, JS_ErrorTerminated);
929 compress_len = ((z_stream*)jcr->pZLIB_compress_workset)->total_out;
930 /* reset zlib stream to be able to begin from scratch again */
931 if ((zstat=deflateReset((z_stream*)jcr->pZLIB_compress_workset)) != Z_OK) {
932 Jmsg(jcr, M_FATAL, 0, _("Compression deflateReset error: %d\n"), zstat);
933 set_jcr_job_status(jcr, JS_ErrorTerminated);
937 Dmsg2(400, "compressed len=%d uncompressed len=%d\n", compress_len,
940 sd->msglen = compress_len; /* set compressed length */
941 cipher_input_len = compress_len;
945 * Note, here we prepend the current record length to the beginning
946 * of the encrypted data. This is because both sparse and compression
947 * restore handling want records returned to them with exactly the
948 * same number of bytes that were processed in the backup handling.
949 * That is, both are block filters rather than a stream. When doing
950 * compression, the compression routines may buffer data, so that for
951 * any one record compressed, when it is decompressed the same size
952 * will not be obtained. Of course, the buffered data eventually comes
953 * out in subsequent crypto_cipher_update() calls or at least
954 * when crypto_cipher_finalize() is called. Unfortunately, this
955 * "feature" of encryption enormously complicates the restore code.
957 if (ff_pkt->flags & FO_ENCRYPT) {
958 uint32_t initial_len = 0;
961 if (ff_pkt->flags & FO_SPARSE) {
962 cipher_input_len += SPARSE_FADDR_SIZE;
965 /* Encrypt the length of the input block */
966 uint8_t packet_len[sizeof(uint32_t)];
968 ser_begin(packet_len, sizeof(uint32_t));
969 ser_uint32(cipher_input_len); /* store data len in begin of buffer */
970 Dmsg1(20, "Encrypt len=%d\n", cipher_input_len);
972 if (!crypto_cipher_update(cipher_ctx, packet_len, sizeof(packet_len),
973 (uint8_t *)jcr->crypto.crypto_buf, &initial_len)) {
974 /* Encryption failed. Shouldn't happen. */
975 Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
979 /* Encrypt the input block */
980 if (crypto_cipher_update(cipher_ctx, cipher_input, cipher_input_len,
981 (uint8_t *)&jcr->crypto.crypto_buf[initial_len], &encrypted_len)) {
982 if ((initial_len + encrypted_len) == 0) {
983 /* No full block of data available, read more data */
986 Dmsg2(400, "encrypted len=%d unencrypted len=%d\n", encrypted_len,
988 sd->msglen = initial_len + encrypted_len; /* set encrypted length */
990 /* Encryption failed. Shouldn't happen. */
991 Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
996 /* Send the buffer to the Storage daemon */
997 if (ff_pkt->flags & FO_SPARSE) {
998 sd->msglen += SPARSE_FADDR_SIZE; /* include fileAddr in size */
1000 sd->msg = wbuf; /* set correct write buffer */
1002 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1006 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
1008 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
1009 sd->msg = msgsave; /* restore read buffer */
1011 } /* end while read file data */
1013 if (sd->msglen < 0) { /* error */
1015 Jmsg(jcr, M_ERROR, 0, _("Read error on file %s. ERR=%s\n"),
1016 ff_pkt->fname, be.bstrerror(ff_pkt->bfd.berrno));
1017 if (jcr->JobErrors++ > 1000) { /* insanity check */
1018 Jmsg(jcr, M_FATAL, 0, _("Too many errors.\n"));
1020 } else if (ff_pkt->flags & FO_ENCRYPT) {
1022 * For encryption, we must call finalize to push out any
1025 if (!crypto_cipher_finalize(cipher_ctx, (uint8_t *)jcr->crypto.crypto_buf,
1027 /* Padding failed. Shouldn't happen. */
1028 Jmsg(jcr, M_FATAL, 0, _("Encryption padding error\n"));
1032 /* Note, on SSL pre-0.9.7, there is always some output */
1033 if (encrypted_len > 0) {
1034 sd->msglen = encrypted_len; /* set encrypted length */
1035 sd->msg = jcr->crypto.crypto_buf; /* set correct write buffer */
1037 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1041 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
1042 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
1043 sd->msg = msgsave; /* restore bnet buffer */
1047 if (!sd->signal(BNET_EOD)) { /* indicate end of file data */
1048 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1053 /* Free the cipher context */
1055 crypto_cipher_free(cipher_ctx);
1060 /* Free the cipher context */
1062 crypto_cipher_free(cipher_ctx);
1065 sd->msg = msgsave; /* restore bnet buffer */
1070 bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream)
1072 BSOCK *sd = jcr->store_bsock;
1073 char attribs[MAXSTRING];
1074 char attribsEx[MAXSTRING];
1077 #ifdef FD_NO_SEND_TEST
1081 Dmsg1(300, "encode_and_send_attrs fname=%s\n", ff_pkt->fname);
1082 /* Find what data stream we will use, then encode the attributes */
1083 if ((data_stream = select_data_stream(ff_pkt)) == STREAM_NONE) {
1084 /* This should not happen */
1085 Jmsg0(jcr, M_FATAL, 0, _("Invalid file flags, no supported data stream type.\n"));
1088 encode_stat(attribs, &ff_pkt->statp, ff_pkt->LinkFI, data_stream);
1090 /* Now possibly extend the attributes */
1091 attr_stream = encode_attribsEx(jcr, attribsEx, ff_pkt);
1093 Dmsg3(300, "File %s\nattribs=%s\nattribsEx=%s\n", ff_pkt->fname, attribs, attribsEx);
1096 jcr->JobFiles++; /* increment number of files sent */
1097 ff_pkt->FileIndex = jcr->JobFiles; /* return FileIndex */
1098 pm_strcpy(jcr->last_fname, ff_pkt->fname);
1102 * Send Attributes header to Storage daemon
1103 * <file-index> <stream> <info>
1105 if (!sd->fsend("%ld %d 0", jcr->JobFiles, attr_stream)) {
1106 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1110 Dmsg1(300, ">stored: attrhdr %s\n", sd->msg);
1113 * Send file attributes to Storage daemon
1116 * Filename (full path)
1117 * Encoded attributes
1118 * Link name (if type==FT_LNK or FT_LNKSAVED)
1119 * Encoded extended-attributes (for Win32)
1121 * For a directory, link is the same as fname, but with trailing
1122 * slash. For a linked file, link is the link.
1124 if (ff_pkt->type != FT_DELETED) { /* already stripped */
1127 if (ff_pkt->type == FT_LNK || ff_pkt->type == FT_LNKSAVED) {
1128 Dmsg2(300, "Link %s to %s\n", ff_pkt->fname, ff_pkt->link);
1129 stat = sd->fsend("%ld %d %s%c%s%c%s%c%s%c", jcr->JobFiles,
1130 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, ff_pkt->link, 0,
1132 } else if (ff_pkt->type == FT_DIREND || ff_pkt->type == FT_REPARSE) {
1133 /* Here link is the canonical filename (i.e. with trailing slash) */
1134 stat = sd->fsend("%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
1135 ff_pkt->type, ff_pkt->link, 0, attribs, 0, 0, attribsEx, 0);
1137 stat = sd->fsend("%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
1138 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, 0, attribsEx, 0);
1140 if (ff_pkt->type != FT_DELETED) {
1141 unstrip_path(ff_pkt);
1144 Dmsg2(300, ">stored: attr len=%d: %s\n", sd->msglen, sd->msg);
1146 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1150 sd->signal(BNET_EOD); /* indicate end of attributes data */
1155 * Do in place strip of path
1157 static bool do_strip(int count, char *in)
1163 /* Copy to first path separator -- Win32 might have c: ... */
1164 while (*in && !IsPathSeparator(*in)) {
1168 numsep++; /* one separator seen */
1169 for (stripped=0; stripped<count && *in; stripped++) {
1170 while (*in && !IsPathSeparator(*in)) {
1171 in++; /* skip chars */
1174 numsep++; /* count separators seen */
1175 in++; /* skip separator */
1179 while (*in) { /* copy to end */
1180 if (IsPathSeparator(*in)) {
1186 Dmsg4(500, "stripped=%d count=%d numsep=%d sep>count=%d\n",
1187 stripped, count, numsep, numsep>count);
1188 return stripped==count && numsep>count;
1192 * If requested strip leading components of the path so that we can
1193 * save file as if it came from a subdirectory. This is most useful
1194 * for dealing with snapshots, by removing the snapshot directory, or
1195 * in handling vendor migrations where files have been restored with
1196 * a vendor product into a subdirectory.
1198 void strip_path(FF_PKT *ff_pkt)
1200 if (!(ff_pkt->flags & FO_STRIPPATH) || ff_pkt->strip_path <= 0) {
1201 Dmsg1(200, "No strip for %s\n", ff_pkt->fname);
1204 if (!ff_pkt->fname_save) {
1205 ff_pkt->fname_save = get_pool_memory(PM_FNAME);
1206 ff_pkt->link_save = get_pool_memory(PM_FNAME);
1208 pm_strcpy(ff_pkt->fname_save, ff_pkt->fname);
1209 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1210 pm_strcpy(ff_pkt->link_save, ff_pkt->link);
1211 Dmsg2(500, "strcpy link_save=%d link=%d\n", strlen(ff_pkt->link_save),
1212 strlen(ff_pkt->link));
1213 sm_check(__FILE__, __LINE__, true);
1217 * Strip path. If it doesn't succeed put it back. If
1218 * it does, and there is a different link string,
1219 * attempt to strip the link. If it fails, back them
1221 * Do not strip symlinks.
1222 * I.e. if either stripping fails don't strip anything.
1224 if (!do_strip(ff_pkt->strip_path, ff_pkt->fname)) {
1225 unstrip_path(ff_pkt);
1228 /* Strip links but not symlinks */
1229 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1230 if (!do_strip(ff_pkt->strip_path, ff_pkt->link)) {
1231 unstrip_path(ff_pkt);
1236 Dmsg3(100, "fname=%s stripped=%s link=%s\n", ff_pkt->fname_save, ff_pkt->fname,
1240 void unstrip_path(FF_PKT *ff_pkt)
1242 if (!(ff_pkt->flags & FO_STRIPPATH) || ff_pkt->strip_path <= 0) {
1245 strcpy(ff_pkt->fname, ff_pkt->fname_save);
1246 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1247 Dmsg2(500, "strcpy link=%s link_save=%s\n", ff_pkt->link,
1249 strcpy(ff_pkt->link, ff_pkt->link_save);
1250 Dmsg2(500, "strcpy link=%d link_save=%d\n", strlen(ff_pkt->link),
1251 strlen(ff_pkt->link_save));
1252 sm_check(__FILE__, __LINE__, true);