2 Bacula® - The Network Backup Solution
4 Copyright (C) 2000-2009 Free Software Foundation Europe e.V.
6 The main author of Bacula is Kern Sibbald, with contributions from
7 many others, a complete list can be found in the file AUTHORS.
8 This program is Free Software; you can redistribute it and/or
9 modify it under the terms of version two of the GNU General Public
10 License as published by the Free Software Foundation and included
13 This program is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
23 Bacula® is a registered trademark of Kern Sibbald.
24 The licensor of Bacula is the Free Software Foundation Europe
25 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
26 Switzerland, email:ftf@fsfeurope.org.
29 * Bacula File Daemon backup.c send file attributes and data
30 * to the Storage daemon.
32 * Kern Sibbald, March MM
42 const bool have_acl = true;
44 const bool have_acl = false;
47 #if defined(HAVE_XATTR)
48 const bool have_xattr = true;
50 const bool have_xattr = false;
53 /* Forward referenced functions */
54 int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level);
55 static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, DIGEST *signature_digest);
56 bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream);
57 static bool crypto_session_start(JCR *jcr);
58 static void crypto_session_end(JCR *jcr);
59 static bool crypto_session_send(JCR *jcr, BSOCK *sd);
62 * Find all the requested files and send them
63 * to the Storage daemon.
65 * Note, we normally carry on a one-way
66 * conversation from this point on with the SD, simply blasting
67 * data to him. To properly know what is going on, we
68 * also run a "heartbeat" monitor which reads the socket and
69 * reacts accordingly (at the moment it has nothing to do
70 * except echo the heartbeat to the Director).
73 bool blast_data_to_storage_daemon(JCR *jcr, char *addr)
77 // TODO landonf: Allow user to specify encryption algorithm
79 sd = jcr->store_bsock;
81 set_jcr_job_status(jcr, JS_Running);
83 Dmsg1(300, "bfiled: opened data connection %d to stored\n", sd->m_fd);
86 CLIENT *client = (CLIENT *)GetNextRes(R_CLIENT, NULL);
90 buf_size = client->max_network_buffer_size;
92 buf_size = 0; /* use default */
94 if (!sd->set_buffer_size(buf_size, BNET_SETBUF_WRITE)) {
95 set_jcr_job_status(jcr, JS_ErrorTerminated);
96 Jmsg(jcr, M_FATAL, 0, _("Cannot set buffer size FD->SD.\n"));
100 jcr->buf_size = sd->msglen;
101 /* Adjust for compression so that output buffer is
102 * 12 bytes + 0.1% larger than input buffer plus 18 bytes.
103 * This gives a bit extra plus room for the sparse addr if any.
104 * Note, we adjust the read size to be smaller so that the
105 * same output buffer can be used without growing it.
107 * The zlib compression workset is initialized here to minimize
108 * the "per file" load. The jcr member is only set, if the init
111 jcr->compress_buf_size = jcr->buf_size + ((jcr->buf_size+999) / 1000) + 30;
112 jcr->compress_buf = get_memory(jcr->compress_buf_size);
115 z_stream *pZlibStream = (z_stream*)malloc(sizeof(z_stream));
117 pZlibStream->zalloc = Z_NULL;
118 pZlibStream->zfree = Z_NULL;
119 pZlibStream->opaque = Z_NULL;
120 pZlibStream->state = Z_NULL;
122 if (deflateInit(pZlibStream, Z_DEFAULT_COMPRESSION) == Z_OK) {
123 jcr->pZLIB_compress_workset = pZlibStream;
130 if (!crypto_session_start(jcr)) {
134 set_find_options((FF_PKT *)jcr->ff, jcr->incremental, jcr->mtime);
136 /* in accurate mode, we overwrite the find_one check function */
138 set_find_changed_function((FF_PKT *)jcr->ff, accurate_check_file);
141 start_heartbeat_monitor(jcr);
144 jcr->acl_data = (acl_data_t *)malloc(sizeof(acl_data_t));
145 memset((caddr_t)jcr->acl_data, 0, sizeof(acl_data_t));
146 jcr->acl_data->content = get_pool_memory(PM_MESSAGE);
150 jcr->xattr_data = (xattr_data_t *)malloc(sizeof(xattr_data_t));
151 memset((caddr_t)jcr->xattr_data, 0, sizeof(xattr_data_t));
152 jcr->xattr_data->content = get_pool_memory(PM_MESSAGE);
155 /* Subroutine save_file() is called for each file */
156 if (!find_files(jcr, (FF_PKT *)jcr->ff, save_file, plugin_save)) {
157 ok = false; /* error */
158 set_jcr_job_status(jcr, JS_ErrorTerminated);
161 if (have_acl && jcr->acl_data->nr_errors > 0) {
162 Jmsg(jcr, M_ERROR, 0, _("Encountered %ld acl errors while doing backup\n"),
163 jcr->acl_data->nr_errors);
165 if (have_xattr && jcr->xattr_data->nr_errors > 0) {
166 Jmsg(jcr, M_ERROR, 0, _("Encountered %ld xattr errors while doing backup\n"),
167 jcr->xattr_data->nr_errors);
170 accurate_finish(jcr); /* send deleted or base file list to SD */
172 stop_heartbeat_monitor(jcr);
174 sd->signal(BNET_EOD); /* end of sending data */
176 if (have_acl && jcr->acl_data) {
177 free_pool_memory(jcr->acl_data->content);
179 jcr->acl_data = NULL;
181 if (have_xattr && jcr->xattr_data) {
182 free_pool_memory(jcr->xattr_data->content);
183 free(jcr->xattr_data);
184 jcr->xattr_data = NULL;
190 if (jcr->compress_buf) {
191 free_pool_memory(jcr->compress_buf);
192 jcr->compress_buf = NULL;
194 if (jcr->pZLIB_compress_workset) {
195 /* Free the zlib stream */
197 deflateEnd((z_stream *)jcr->pZLIB_compress_workset);
199 free (jcr->pZLIB_compress_workset);
200 jcr->pZLIB_compress_workset = NULL;
202 crypto_session_end(jcr);
205 Dmsg1(100, "end blast_data ok=%d\n", ok);
209 static bool crypto_session_start(JCR *jcr)
211 crypto_cipher_t cipher = CRYPTO_CIPHER_AES_128_CBC;
214 * Create encryption session data and a cached, DER-encoded session data
215 * structure. We use a single session key for each backup, so we'll encode
216 * the session data only once.
218 if (jcr->crypto.pki_encrypt) {
221 /* Create per-job session encryption context */
222 jcr->crypto.pki_session = crypto_session_new(cipher, jcr->crypto.pki_recipients);
224 /* Get the session data size */
225 if (!crypto_session_encode(jcr->crypto.pki_session, (uint8_t *)0, &size)) {
226 Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n"));
230 /* Allocate buffer */
231 jcr->crypto.pki_session_encoded = get_memory(size);
233 /* Encode session data */
234 if (!crypto_session_encode(jcr->crypto.pki_session, (uint8_t *)jcr->crypto.pki_session_encoded, &size)) {
235 Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n"));
239 /* ... and store the encoded size */
240 jcr->crypto.pki_session_encoded_size = size;
242 /* Allocate the encryption/decryption buffer */
243 jcr->crypto.crypto_buf = get_memory(CRYPTO_CIPHER_MAX_BLOCK_SIZE);
248 static void crypto_session_end(JCR *jcr)
250 if (jcr->crypto.crypto_buf) {
251 free_pool_memory(jcr->crypto.crypto_buf);
252 jcr->crypto.crypto_buf = NULL;
254 if (jcr->crypto.pki_session) {
255 crypto_session_free(jcr->crypto.pki_session);
257 if (jcr->crypto.pki_session_encoded) {
258 free_pool_memory(jcr->crypto.pki_session_encoded);
259 jcr->crypto.pki_session_encoded = NULL;
263 static bool crypto_session_send(JCR *jcr, BSOCK *sd)
267 /* Send our header */
268 Dmsg2(100, "Send hdr fi=%ld stream=%d\n", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA);
269 sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA);
272 sd->msg = jcr->crypto.pki_session_encoded;
273 sd->msglen = jcr->crypto.pki_session_encoded_size;
274 jcr->JobBytes += sd->msglen;
276 Dmsg1(100, "Send data len=%d\n", sd->msglen);
279 sd->signal(BNET_EOD);
285 * Called here by find() for each file included.
286 * This is a callback. The original is find_files() above.
288 * Send the file and its data to the Storage daemon.
292 * -1 to ignore file/directory (not used here)
294 int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level)
296 bool do_read = false;
297 int stat, data_stream;
299 DIGEST *digest = NULL;
300 DIGEST *signing_digest = NULL;
301 int digest_stream = STREAM_NONE;
302 SIGNATURE *sig = NULL;
303 bool has_file_data = false;
304 // TODO landonf: Allow the user to specify the digest algorithm
306 crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA256;
308 crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA1;
310 BSOCK *sd = jcr->store_bsock;
312 if (job_canceled(jcr)) {
316 jcr->num_files_examined++; /* bump total file count */
318 switch (ff_pkt->type) {
319 case FT_LNKSAVED: /* Hard linked, file already saved */
320 Dmsg2(130, "FT_LNKSAVED hard link: %s => %s\n", ff_pkt->fname, ff_pkt->link);
323 Dmsg1(130, "FT_REGE saving: %s\n", ff_pkt->fname);
324 has_file_data = true;
327 Dmsg1(130, "FT_REG saving: %s\n", ff_pkt->fname);
328 has_file_data = true;
331 Dmsg2(130, "FT_LNK saving: %s -> %s\n", ff_pkt->fname, ff_pkt->link);
334 jcr->num_files_examined--; /* correct file count */
335 return 1; /* not used */
337 Jmsg(jcr, M_INFO, 1, _(" Recursion turned off. Will not descend from %s into %s\n"),
338 ff_pkt->top_fname, ff_pkt->fname);
339 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
342 /* Suppress message for /dev filesystems */
343 if (!is_in_fileset(ff_pkt)) {
344 Jmsg(jcr, M_INFO, 1, _(" %s is a different filesystem. Will not descend from %s into %s\n"),
345 ff_pkt->fname, ff_pkt->top_fname, ff_pkt->fname);
347 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
350 Jmsg(jcr, M_INFO, 1, _(" Disallowed filesystem. Will not descend from %s into %s\n"),
351 ff_pkt->top_fname, ff_pkt->fname);
352 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
355 Jmsg(jcr, M_INFO, 1, _(" Disallowed drive type. Will not descend into %s\n"),
360 Dmsg1(130, "FT_DIREND: %s\n", ff_pkt->link);
363 Dmsg1(130, "FT_SPEC saving: %s\n", ff_pkt->fname);
364 if (S_ISSOCK(ff_pkt->statp.st_mode)) {
365 Jmsg(jcr, M_SKIPPED, 1, _(" Socket file skipped: %s\n"), ff_pkt->fname);
370 Dmsg1(130, "FT_RAW saving: %s\n", ff_pkt->fname);
371 has_file_data = true;
374 Dmsg1(130, "FT_FIFO saving: %s\n", ff_pkt->fname);
378 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not access \"%s\": ERR=%s\n"), ff_pkt->fname,
379 be.bstrerror(ff_pkt->ff_errno));
385 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not follow link \"%s\": ERR=%s\n"),
386 ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
392 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not stat \"%s\": ERR=%s\n"), ff_pkt->fname,
393 be.bstrerror(ff_pkt->ff_errno));
399 Jmsg(jcr, M_SKIPPED, 1, _(" Unchanged file skipped: %s\n"), ff_pkt->fname);
402 Jmsg(jcr, M_NOTSAVED, 0, _(" Archive file not saved: %s\n"), ff_pkt->fname);
406 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not open directory \"%s\": ERR=%s\n"),
407 ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
412 Jmsg(jcr, M_NOTSAVED, 0, _(" Unknown file type %d; not saved: %s\n"),
413 ff_pkt->type, ff_pkt->fname);
418 Dmsg1(130, "bfiled: sending %s to stored\n", ff_pkt->fname);
420 /* Digests and encryption are only useful if there's file data */
423 * Setup for digest handling. If this fails, the digest will be set to NULL
424 * and not used. Note, the digest (file hash) can be any one of the four
427 * The signing digest is a single algorithm depending on
428 * whether or not we have SHA2.
429 * ****FIXME**** the signing algoritm should really be
430 * determined a different way!!!!!! What happens if
431 * sha2 was available during backup but not restore?
433 if (ff_pkt->flags & FO_MD5) {
434 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_MD5);
435 digest_stream = STREAM_MD5_DIGEST;
437 } else if (ff_pkt->flags & FO_SHA1) {
438 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA1);
439 digest_stream = STREAM_SHA1_DIGEST;
441 } else if (ff_pkt->flags & FO_SHA256) {
442 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA256);
443 digest_stream = STREAM_SHA256_DIGEST;
445 } else if (ff_pkt->flags & FO_SHA512) {
446 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA512);
447 digest_stream = STREAM_SHA512_DIGEST;
450 /* Did digest initialization fail? */
451 if (digest_stream != STREAM_NONE && digest == NULL) {
452 Jmsg(jcr, M_WARNING, 0, _("%s digest initialization failed\n"),
453 stream_to_ascii(digest_stream));
457 * Set up signature digest handling. If this fails, the signature digest
458 * will be set to NULL and not used.
460 /* TODO landonf: We should really only calculate the digest once, for
461 * both verification and signing.
463 if (jcr->crypto.pki_sign) {
464 signing_digest = crypto_digest_new(jcr, signing_algorithm);
466 /* Full-stop if a failure occurred initializing the signature digest */
467 if (signing_digest == NULL) {
468 Jmsg(jcr, M_NOTSAVED, 0, _("%s signature digest initialization failed\n"),
469 stream_to_ascii(signing_algorithm));
475 /* Enable encryption */
476 if (jcr->crypto.pki_encrypt) {
477 ff_pkt->flags |= FO_ENCRYPT;
481 /* Initialize the file descriptor we use for data and other streams. */
483 if (ff_pkt->flags & FO_PORTABLE) {
484 set_portable_backup(&ff_pkt->bfd); /* disable Win32 BackupRead() */
486 if (ff_pkt->cmd_plugin) {
487 if (!set_cmd_plugin(&ff_pkt->bfd, jcr)) {
490 send_plugin_name(jcr, sd, true); /* signal start of plugin data */
493 /* Send attributes -- must be done after binit() */
494 if (!encode_and_send_attributes(jcr, ff_pkt, data_stream)) {
498 /* Set up the encryption context and send the session data to the SD */
499 if (has_file_data && jcr->crypto.pki_encrypt) {
500 if (!crypto_session_send(jcr, sd)) {
506 * Open any file with data that we intend to save, then save it.
508 * Note, if is_win32_backup, we must open the Directory so that
509 * the BackupRead will save its permissions and ownership streams.
511 if (ff_pkt->type != FT_LNKSAVED && S_ISREG(ff_pkt->statp.st_mode)) {
513 do_read = !is_portable_backup(&ff_pkt->bfd) || ff_pkt->statp.st_size > 0;
515 do_read = ff_pkt->statp.st_size > 0;
517 } else if (ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO ||
518 ff_pkt->type == FT_REPARSE ||
519 (!is_portable_backup(&ff_pkt->bfd) && ff_pkt->type == FT_DIREND)) {
522 if (ff_pkt->cmd_plugin) {
526 Dmsg1(400, "do_read=%d\n", do_read);
530 if (ff_pkt->type == FT_FIFO) {
531 tid = start_thread_timer(jcr, pthread_self(), 60);
535 int noatime = ff_pkt->flags & FO_NOATIME ? O_NOATIME : 0;
536 ff_pkt->bfd.reparse_point = ff_pkt->type == FT_REPARSE;
537 if (bopen(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY | noatime, 0) < 0) {
538 ff_pkt->ff_errno = errno;
540 Jmsg(jcr, M_NOTSAVED, 0, _(" Cannot open \"%s\": ERR=%s.\n"), ff_pkt->fname,
544 stop_thread_timer(tid);
550 stop_thread_timer(tid);
554 stat = send_data(jcr, data_stream, ff_pkt, digest, signing_digest);
556 if (ff_pkt->flags & FO_CHKCHANGES) {
557 has_file_changed(jcr, ff_pkt);
560 bclose(&ff_pkt->bfd);
567 #ifdef HAVE_DARWIN_OS
568 /* Regular files can have resource forks and Finder Info */
569 if (ff_pkt->type != FT_LNKSAVED && (S_ISREG(ff_pkt->statp.st_mode) &&
570 ff_pkt->flags & FO_HFSPLUS)) {
571 if (ff_pkt->hfsinfo.rsrclength > 0) {
574 if (!bopen_rsrc(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0) < 0) {
575 ff_pkt->ff_errno = errno;
577 Jmsg(jcr, M_NOTSAVED, -1, _(" Cannot open resource fork for \"%s\": ERR=%s.\n"),
578 ff_pkt->fname, be.bstrerror());
580 if (is_bopen(&ff_pkt->bfd)) {
581 bclose(&ff_pkt->bfd);
585 flags = ff_pkt->flags;
586 ff_pkt->flags &= ~(FO_GZIP|FO_SPARSE);
587 if (flags & FO_ENCRYPT) {
588 rsrc_stream = STREAM_ENCRYPTED_MACOS_FORK_DATA;
590 rsrc_stream = STREAM_MACOS_FORK_DATA;
592 stat = send_data(jcr, rsrc_stream, ff_pkt, digest, signing_digest);
593 ff_pkt->flags = flags;
594 bclose(&ff_pkt->bfd);
600 Dmsg1(300, "Saving Finder Info for \"%s\"\n", ff_pkt->fname);
601 sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_HFSPLUS_ATTRIBUTES);
602 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
603 pm_memcpy(sd->msg, ff_pkt->hfsinfo.fndrinfo, 32);
606 crypto_digest_update(digest, (uint8_t *)sd->msg, sd->msglen);
608 if (signing_digest) {
609 crypto_digest_update(signing_digest, (uint8_t *)sd->msg, sd->msglen);
612 sd->signal(BNET_EOD);
617 * Save ACLs when requested and available for anything not being a symlink and not being a plugin.
620 if (ff_pkt->flags & FO_ACL && ff_pkt->type != FT_LNK && !ff_pkt->cmd_plugin) {
621 switch (build_acl_streams(jcr, ff_pkt)) {
622 case bacl_exit_fatal:
624 case bacl_exit_error:
626 * Non-fatal errors, count them and when the number is under ACL_REPORT_ERR_MAX_PER_JOB
627 * print the error message set by the lower level routine in jcr->errmsg.
629 if (jcr->acl_data->nr_errors < ACL_REPORT_ERR_MAX_PER_JOB) {
630 Jmsg(jcr, M_ERROR, 0, "%s", jcr->errmsg);
632 jcr->acl_data->nr_errors++;
641 * Save Extended Attributes when requested and available for all files not being a plugin.
644 if (ff_pkt->flags & FO_XATTR && !ff_pkt->cmd_plugin) {
645 switch (build_xattr_streams(jcr, ff_pkt)) {
646 case bxattr_exit_fatal:
648 case bxattr_exit_error:
650 * Non-fatal errors, count them and when the number is under XATTR_REPORT_ERR_MAX_PER_JOB
651 * print the error message set by the lower level routine in jcr->errmsg.
653 if (jcr->xattr_data->nr_errors < XATTR_REPORT_ERR_MAX_PER_JOB) {
654 Jmsg(jcr, M_ERROR, 0, "%s", jcr->errmsg);
656 jcr->xattr_data->nr_errors++;
664 /* Terminate the signing digest and send it to the Storage daemon */
665 if (signing_digest) {
668 if ((sig = crypto_sign_new(jcr)) == NULL) {
669 Jmsg(jcr, M_FATAL, 0, _("Failed to allocate memory for crypto signature.\n"));
673 if (!crypto_sign_add_signer(sig, signing_digest, jcr->crypto.pki_keypair)) {
674 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
678 /* Get signature size */
679 if (!crypto_sign_encode(sig, NULL, &size)) {
680 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
684 /* Grow the bsock buffer to fit our message if necessary */
685 if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
686 sd->msg = realloc_pool_memory(sd->msg, size);
689 /* Send our header */
690 sd->fsend("%ld %ld 0", jcr->JobFiles, STREAM_SIGNED_DIGEST);
691 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
693 /* Encode signature data */
694 if (!crypto_sign_encode(sig, (uint8_t *)sd->msg, &size)) {
695 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
701 sd->signal(BNET_EOD); /* end of checksum */
704 /* Terminate any digest and send it to Storage daemon */
708 sd->fsend("%ld %d 0", jcr->JobFiles, digest_stream);
709 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
711 size = CRYPTO_DIGEST_MAX_SIZE;
713 /* Grow the bsock buffer to fit our message if necessary */
714 if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
715 sd->msg = realloc_pool_memory(sd->msg, size);
718 if (!crypto_digest_finalize(digest, (uint8_t *)sd->msg, &size)) {
719 Jmsg(jcr, M_FATAL, 0, _("An error occurred finalizing signing the stream.\n"));
725 sd->signal(BNET_EOD); /* end of checksum */
727 if (ff_pkt->cmd_plugin) {
728 send_plugin_name(jcr, sd, false); /* signal end of plugin data */
732 rtnstat = 1; /* good return */
736 crypto_digest_free(digest);
738 if (signing_digest) {
739 crypto_digest_free(signing_digest);
742 crypto_sign_free(sig);
748 * Send data read from an already open file descriptor.
750 * We return 1 on sucess and 0 on errors.
753 * We use ff_pkt->statp.st_size when FO_SPARSE to know when to stop
755 * Currently this is not a problem as the only other stream, resource forks,
756 * are not handled as sparse files.
758 static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest,
759 DIGEST *signing_digest)
761 BSOCK *sd = jcr->store_bsock;
762 uint64_t fileAddr = 0; /* file address */
764 int32_t rsize = jcr->buf_size; /* read buffer size */
766 CIPHER_CONTEXT *cipher_ctx = NULL; /* Quell bogus uninitialized warnings */
767 const uint8_t *cipher_input;
768 uint32_t cipher_input_len;
769 uint32_t cipher_block_size;
770 uint32_t encrypted_len;
771 #ifdef FD_NO_SEND_TEST
776 rbuf = sd->msg; /* read buffer */
777 wbuf = sd->msg; /* write buffer */
778 cipher_input = (uint8_t *)rbuf; /* encrypt uncompressed data */
780 Dmsg1(300, "Saving data, type=%d\n", ff_pkt->type);
783 uLong compress_len = 0;
784 uLong max_compress_len = 0;
785 const Bytef *cbuf = NULL;
788 if (ff_pkt->flags & FO_GZIP) {
789 if (ff_pkt->flags & FO_SPARSE) {
790 cbuf = (Bytef *)jcr->compress_buf + SPARSE_FADDR_SIZE;
791 max_compress_len = jcr->compress_buf_size - SPARSE_FADDR_SIZE;
793 cbuf = (Bytef *)jcr->compress_buf;
794 max_compress_len = jcr->compress_buf_size; /* set max length */
796 wbuf = jcr->compress_buf; /* compressed output here */
797 cipher_input = (uint8_t *)jcr->compress_buf; /* encrypt compressed data */
800 * Only change zlib parameters if there is no pending operation.
801 * This should never happen as deflatereset is called after each
805 if (((z_stream*)jcr->pZLIB_compress_workset)->total_in == 0) {
806 /* set gzip compression level - must be done per file */
807 if ((zstat=deflateParams((z_stream*)jcr->pZLIB_compress_workset,
808 ff_pkt->GZIP_level, Z_DEFAULT_STRATEGY)) != Z_OK) {
809 Jmsg(jcr, M_FATAL, 0, _("Compression deflateParams error: %d\n"), zstat);
810 set_jcr_job_status(jcr, JS_ErrorTerminated);
816 const uint32_t max_compress_len = 0;
819 if (ff_pkt->flags & FO_ENCRYPT) {
820 if (ff_pkt->flags & FO_SPARSE) {
821 Jmsg0(jcr, M_FATAL, 0, _("Encrypting sparse data not supported.\n"));
824 /* Allocate the cipher context */
825 if ((cipher_ctx = crypto_cipher_new(jcr->crypto.pki_session, true,
826 &cipher_block_size)) == NULL) {
827 /* Shouldn't happen! */
828 Jmsg0(jcr, M_FATAL, 0, _("Failed to initialize encryption context.\n"));
833 * Grow the crypto buffer, if necessary.
834 * crypto_cipher_update() will buffer up to (cipher_block_size - 1).
835 * We grow crypto_buf to the maximum number of blocks that
836 * could be returned for the given read buffer size.
837 * (Using the larger of either rsize or max_compress_len)
839 jcr->crypto.crypto_buf = check_pool_memory_size(jcr->crypto.crypto_buf,
840 (MAX(rsize + (int)sizeof(uint32_t), (int32_t)max_compress_len) +
841 cipher_block_size - 1) / cipher_block_size * cipher_block_size);
843 wbuf = jcr->crypto.crypto_buf; /* Encrypted, possibly compressed output here. */
847 * Send Data header to Storage daemon
848 * <file-index> <stream> <info>
850 if (!sd->fsend("%ld %d 0", jcr->JobFiles, stream)) {
851 if (!job_canceled(jcr)) {
852 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
857 Dmsg1(300, ">stored: datahdr %s\n", sd->msg);
860 * Make space at beginning of buffer for fileAddr because this
861 * same buffer will be used for writing if compression is off.
863 if (ff_pkt->flags & FO_SPARSE) {
864 rbuf += SPARSE_FADDR_SIZE;
865 rsize -= SPARSE_FADDR_SIZE;
866 #ifdef HAVE_FREEBSD_OS
868 * To read FreeBSD partitions, the read size must be
871 rsize = (rsize/512) * 512;
875 /* a RAW device read on win32 only works if the buffer is a multiple of 512 */
877 if (S_ISBLK(ff_pkt->statp.st_mode))
878 rsize = (rsize/512) * 512;
884 while ((sd->msglen=(uint32_t)bread(&ff_pkt->bfd, rbuf, rsize)) > 0) {
886 /* Check for sparse blocks */
887 if (ff_pkt->flags & FO_SPARSE) {
889 bool allZeros = false;
890 if ((sd->msglen == rsize &&
891 fileAddr+sd->msglen < (uint64_t)ff_pkt->statp.st_size) ||
892 ((ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO) &&
893 (uint64_t)ff_pkt->statp.st_size == 0)) {
894 allZeros = is_buf_zero(rbuf, rsize);
897 /* Put file address as first data in buffer */
898 ser_begin(wbuf, SPARSE_FADDR_SIZE);
899 ser_uint64(fileAddr); /* store fileAddr in begin of buffer */
901 fileAddr += sd->msglen; /* update file address */
902 /* Skip block of all zeros */
904 continue; /* skip block of zeros */
908 jcr->ReadBytes += sd->msglen; /* count bytes read */
910 /* Uncompressed cipher input length */
911 cipher_input_len = sd->msglen;
913 /* Update checksum if requested */
915 crypto_digest_update(digest, (uint8_t *)rbuf, sd->msglen);
918 /* Update signing digest if requested */
919 if (signing_digest) {
920 crypto_digest_update(signing_digest, (uint8_t *)rbuf, sd->msglen);
924 /* Do compression if turned on */
925 if (ff_pkt->flags & FO_GZIP && jcr->pZLIB_compress_workset) {
926 Dmsg3(400, "cbuf=0x%x rbuf=0x%x len=%u\n", cbuf, rbuf, sd->msglen);
928 ((z_stream*)jcr->pZLIB_compress_workset)->next_in = (Bytef *)rbuf;
929 ((z_stream*)jcr->pZLIB_compress_workset)->avail_in = sd->msglen;
930 ((z_stream*)jcr->pZLIB_compress_workset)->next_out = (Bytef *)cbuf;
931 ((z_stream*)jcr->pZLIB_compress_workset)->avail_out = max_compress_len;
933 if ((zstat=deflate((z_stream*)jcr->pZLIB_compress_workset, Z_FINISH)) != Z_STREAM_END) {
934 Jmsg(jcr, M_FATAL, 0, _("Compression deflate error: %d\n"), zstat);
935 set_jcr_job_status(jcr, JS_ErrorTerminated);
938 compress_len = ((z_stream*)jcr->pZLIB_compress_workset)->total_out;
939 /* reset zlib stream to be able to begin from scratch again */
940 if ((zstat=deflateReset((z_stream*)jcr->pZLIB_compress_workset)) != Z_OK) {
941 Jmsg(jcr, M_FATAL, 0, _("Compression deflateReset error: %d\n"), zstat);
942 set_jcr_job_status(jcr, JS_ErrorTerminated);
946 Dmsg2(400, "compressed len=%d uncompressed len=%d\n", compress_len,
949 sd->msglen = compress_len; /* set compressed length */
950 cipher_input_len = compress_len;
954 * Note, here we prepend the current record length to the beginning
955 * of the encrypted data. This is because both sparse and compression
956 * restore handling want records returned to them with exactly the
957 * same number of bytes that were processed in the backup handling.
958 * That is, both are block filters rather than a stream. When doing
959 * compression, the compression routines may buffer data, so that for
960 * any one record compressed, when it is decompressed the same size
961 * will not be obtained. Of course, the buffered data eventually comes
962 * out in subsequent crypto_cipher_update() calls or at least
963 * when crypto_cipher_finalize() is called. Unfortunately, this
964 * "feature" of encryption enormously complicates the restore code.
966 if (ff_pkt->flags & FO_ENCRYPT) {
967 uint32_t initial_len = 0;
970 if (ff_pkt->flags & FO_SPARSE) {
971 cipher_input_len += SPARSE_FADDR_SIZE;
974 /* Encrypt the length of the input block */
975 uint8_t packet_len[sizeof(uint32_t)];
977 ser_begin(packet_len, sizeof(uint32_t));
978 ser_uint32(cipher_input_len); /* store data len in begin of buffer */
979 Dmsg1(20, "Encrypt len=%d\n", cipher_input_len);
981 if (!crypto_cipher_update(cipher_ctx, packet_len, sizeof(packet_len),
982 (uint8_t *)jcr->crypto.crypto_buf, &initial_len)) {
983 /* Encryption failed. Shouldn't happen. */
984 Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
988 /* Encrypt the input block */
989 if (crypto_cipher_update(cipher_ctx, cipher_input, cipher_input_len,
990 (uint8_t *)&jcr->crypto.crypto_buf[initial_len], &encrypted_len)) {
991 if ((initial_len + encrypted_len) == 0) {
992 /* No full block of data available, read more data */
995 Dmsg2(400, "encrypted len=%d unencrypted len=%d\n", encrypted_len,
997 sd->msglen = initial_len + encrypted_len; /* set encrypted length */
999 /* Encryption failed. Shouldn't happen. */
1000 Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
1005 /* Send the buffer to the Storage daemon */
1006 if (ff_pkt->flags & FO_SPARSE) {
1007 sd->msglen += SPARSE_FADDR_SIZE; /* include fileAddr in size */
1009 sd->msg = wbuf; /* set correct write buffer */
1011 if (!job_canceled(jcr)) {
1012 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1017 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
1019 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
1020 sd->msg = msgsave; /* restore read buffer */
1022 } /* end while read file data */
1024 if (sd->msglen < 0) { /* error */
1026 Jmsg(jcr, M_ERROR, 0, _("Read error on file %s. ERR=%s\n"),
1027 ff_pkt->fname, be.bstrerror(ff_pkt->bfd.berrno));
1028 if (jcr->JobErrors++ > 1000) { /* insanity check */
1029 Jmsg(jcr, M_FATAL, 0, _("Too many errors.\n"));
1031 } else if (ff_pkt->flags & FO_ENCRYPT) {
1033 * For encryption, we must call finalize to push out any
1036 if (!crypto_cipher_finalize(cipher_ctx, (uint8_t *)jcr->crypto.crypto_buf,
1038 /* Padding failed. Shouldn't happen. */
1039 Jmsg(jcr, M_FATAL, 0, _("Encryption padding error\n"));
1043 /* Note, on SSL pre-0.9.7, there is always some output */
1044 if (encrypted_len > 0) {
1045 sd->msglen = encrypted_len; /* set encrypted length */
1046 sd->msg = jcr->crypto.crypto_buf; /* set correct write buffer */
1048 if (!job_canceled(jcr)) {
1049 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1054 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
1055 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
1056 sd->msg = msgsave; /* restore bnet buffer */
1060 if (!sd->signal(BNET_EOD)) { /* indicate end of file data */
1061 if (!job_canceled(jcr)) {
1062 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1068 /* Free the cipher context */
1070 crypto_cipher_free(cipher_ctx);
1075 /* Free the cipher context */
1077 crypto_cipher_free(cipher_ctx);
1080 sd->msg = msgsave; /* restore bnet buffer */
1085 bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream)
1087 BSOCK *sd = jcr->store_bsock;
1088 char attribs[MAXSTRING];
1089 char attribsEx[MAXSTRING];
1092 #ifdef FD_NO_SEND_TEST
1096 Dmsg1(300, "encode_and_send_attrs fname=%s\n", ff_pkt->fname);
1097 /* Find what data stream we will use, then encode the attributes */
1098 if ((data_stream = select_data_stream(ff_pkt)) == STREAM_NONE) {
1099 /* This should not happen */
1100 Jmsg0(jcr, M_FATAL, 0, _("Invalid file flags, no supported data stream type.\n"));
1103 encode_stat(attribs, &ff_pkt->statp, ff_pkt->LinkFI, data_stream);
1105 /* Now possibly extend the attributes */
1106 attr_stream = encode_attribsEx(jcr, attribsEx, ff_pkt);
1108 Dmsg3(300, "File %s\nattribs=%s\nattribsEx=%s\n", ff_pkt->fname, attribs, attribsEx);
1111 jcr->JobFiles++; /* increment number of files sent */
1112 ff_pkt->FileIndex = jcr->JobFiles; /* return FileIndex */
1113 pm_strcpy(jcr->last_fname, ff_pkt->fname);
1117 * Send Attributes header to Storage daemon
1118 * <file-index> <stream> <info>
1120 if (!sd->fsend("%ld %d 0", jcr->JobFiles, attr_stream)) {
1121 if (!job_canceled(jcr)) {
1122 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1127 Dmsg1(300, ">stored: attrhdr %s\n", sd->msg);
1130 * Send file attributes to Storage daemon
1133 * Filename (full path)
1134 * Encoded attributes
1135 * Link name (if type==FT_LNK or FT_LNKSAVED)
1136 * Encoded extended-attributes (for Win32)
1138 * For a directory, link is the same as fname, but with trailing
1139 * slash. For a linked file, link is the link.
1141 if (ff_pkt->type != FT_DELETED) { /* already stripped */
1144 if (ff_pkt->type == FT_LNK || ff_pkt->type == FT_LNKSAVED) {
1145 Dmsg2(300, "Link %s to %s\n", ff_pkt->fname, ff_pkt->link);
1146 stat = sd->fsend("%ld %d %s%c%s%c%s%c%s%c", jcr->JobFiles,
1147 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, ff_pkt->link, 0,
1149 } else if (ff_pkt->type == FT_DIREND || ff_pkt->type == FT_REPARSE) {
1150 /* Here link is the canonical filename (i.e. with trailing slash) */
1151 stat = sd->fsend("%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
1152 ff_pkt->type, ff_pkt->link, 0, attribs, 0, 0, attribsEx, 0);
1154 stat = sd->fsend("%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
1155 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, 0, attribsEx, 0);
1157 if (ff_pkt->type != FT_DELETED) {
1158 unstrip_path(ff_pkt);
1161 Dmsg2(300, ">stored: attr len=%d: %s\n", sd->msglen, sd->msg);
1163 if (!job_canceled(jcr)) {
1164 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1169 sd->signal(BNET_EOD); /* indicate end of attributes data */
1174 * Do in place strip of path
1176 static bool do_strip(int count, char *in)
1182 /* Copy to first path separator -- Win32 might have c: ... */
1183 while (*in && !IsPathSeparator(*in)) {
1187 numsep++; /* one separator seen */
1188 for (stripped=0; stripped<count && *in; stripped++) {
1189 while (*in && !IsPathSeparator(*in)) {
1190 in++; /* skip chars */
1193 numsep++; /* count separators seen */
1194 in++; /* skip separator */
1198 while (*in) { /* copy to end */
1199 if (IsPathSeparator(*in)) {
1205 Dmsg4(500, "stripped=%d count=%d numsep=%d sep>count=%d\n",
1206 stripped, count, numsep, numsep>count);
1207 return stripped==count && numsep>count;
1211 * If requested strip leading components of the path so that we can
1212 * save file as if it came from a subdirectory. This is most useful
1213 * for dealing with snapshots, by removing the snapshot directory, or
1214 * in handling vendor migrations where files have been restored with
1215 * a vendor product into a subdirectory.
1217 void strip_path(FF_PKT *ff_pkt)
1219 if (!(ff_pkt->flags & FO_STRIPPATH) || ff_pkt->strip_path <= 0) {
1220 Dmsg1(200, "No strip for %s\n", ff_pkt->fname);
1223 if (!ff_pkt->fname_save) {
1224 ff_pkt->fname_save = get_pool_memory(PM_FNAME);
1225 ff_pkt->link_save = get_pool_memory(PM_FNAME);
1227 pm_strcpy(ff_pkt->fname_save, ff_pkt->fname);
1228 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1229 pm_strcpy(ff_pkt->link_save, ff_pkt->link);
1230 Dmsg2(500, "strcpy link_save=%d link=%d\n", strlen(ff_pkt->link_save),
1231 strlen(ff_pkt->link));
1232 sm_check(__FILE__, __LINE__, true);
1236 * Strip path. If it doesn't succeed put it back. If
1237 * it does, and there is a different link string,
1238 * attempt to strip the link. If it fails, back them
1240 * Do not strip symlinks.
1241 * I.e. if either stripping fails don't strip anything.
1243 if (!do_strip(ff_pkt->strip_path, ff_pkt->fname)) {
1244 unstrip_path(ff_pkt);
1247 /* Strip links but not symlinks */
1248 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1249 if (!do_strip(ff_pkt->strip_path, ff_pkt->link)) {
1250 unstrip_path(ff_pkt);
1255 Dmsg3(100, "fname=%s stripped=%s link=%s\n", ff_pkt->fname_save, ff_pkt->fname,
1259 void unstrip_path(FF_PKT *ff_pkt)
1261 if (!(ff_pkt->flags & FO_STRIPPATH) || ff_pkt->strip_path <= 0) {
1264 strcpy(ff_pkt->fname, ff_pkt->fname_save);
1265 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1266 Dmsg2(500, "strcpy link=%s link_save=%s\n", ff_pkt->link,
1268 strcpy(ff_pkt->link, ff_pkt->link_save);
1269 Dmsg2(500, "strcpy link=%d link_save=%d\n", strlen(ff_pkt->link),
1270 strlen(ff_pkt->link_save));
1271 sm_check(__FILE__, __LINE__, true);