2 Bacula® - The Network Backup Solution
4 Copyright (C) 2000-2009 Free Software Foundation Europe e.V.
6 The main author of Bacula is Kern Sibbald, with contributions from
7 many others, a complete list can be found in the file AUTHORS.
8 This program is Free Software; you can redistribute it and/or
9 modify it under the terms of version two of the GNU General Public
10 License as published by the Free Software Foundation and included
13 This program is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
23 Bacula® is a registered trademark of Kern Sibbald.
24 The licensor of Bacula is the Free Software Foundation Europe
25 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
26 Switzerland, email:ftf@fsfeurope.org.
29 * Bacula File Daemon backup.c send file attributes and data
30 * to the Storage daemon.
32 * Kern Sibbald, March MM
42 const bool have_darwin_os = true;
44 const bool have_darwin_os = false;
48 const bool have_acl = true;
50 const bool have_acl = false;
53 #if defined(HAVE_XATTR)
54 const bool have_xattr = true;
56 const bool have_xattr = false;
59 /* Forward referenced functions */
60 int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level);
61 static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, DIGEST *signature_digest);
62 bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream);
63 static bool crypto_session_start(JCR *jcr);
64 static void crypto_session_end(JCR *jcr);
65 static bool crypto_session_send(JCR *jcr, BSOCK *sd);
68 * Find all the requested files and send them
69 * to the Storage daemon.
71 * Note, we normally carry on a one-way
72 * conversation from this point on with the SD, simply blasting
73 * data to him. To properly know what is going on, we
74 * also run a "heartbeat" monitor which reads the socket and
75 * reacts accordingly (at the moment it has nothing to do
76 * except echo the heartbeat to the Director).
79 bool blast_data_to_storage_daemon(JCR *jcr, char *addr)
83 // TODO landonf: Allow user to specify encryption algorithm
85 sd = jcr->store_bsock;
87 set_jcr_job_status(jcr, JS_Running);
89 Dmsg1(300, "bfiled: opened data connection %d to stored\n", sd->m_fd);
92 CLIENT *client = (CLIENT *)GetNextRes(R_CLIENT, NULL);
96 buf_size = client->max_network_buffer_size;
98 buf_size = 0; /* use default */
100 if (!sd->set_buffer_size(buf_size, BNET_SETBUF_WRITE)) {
101 set_jcr_job_status(jcr, JS_ErrorTerminated);
102 Jmsg(jcr, M_FATAL, 0, _("Cannot set buffer size FD->SD.\n"));
106 jcr->buf_size = sd->msglen;
107 /* Adjust for compression so that output buffer is
108 * 12 bytes + 0.1% larger than input buffer plus 18 bytes.
109 * This gives a bit extra plus room for the sparse addr if any.
110 * Note, we adjust the read size to be smaller so that the
111 * same output buffer can be used without growing it.
113 * The zlib compression workset is initialized here to minimize
114 * the "per file" load. The jcr member is only set, if the init
117 jcr->compress_buf_size = jcr->buf_size + ((jcr->buf_size+999) / 1000) + 30;
118 jcr->compress_buf = get_memory(jcr->compress_buf_size);
121 z_stream *pZlibStream = (z_stream*)malloc(sizeof(z_stream));
123 pZlibStream->zalloc = Z_NULL;
124 pZlibStream->zfree = Z_NULL;
125 pZlibStream->opaque = Z_NULL;
126 pZlibStream->state = Z_NULL;
128 if (deflateInit(pZlibStream, Z_DEFAULT_COMPRESSION) == Z_OK) {
129 jcr->pZLIB_compress_workset = pZlibStream;
136 if (!crypto_session_start(jcr)) {
140 set_find_options((FF_PKT *)jcr->ff, jcr->incremental, jcr->mtime);
142 /* in accurate mode, we overwrite the find_one check function */
144 set_find_changed_function((FF_PKT *)jcr->ff, accurate_check_file);
147 start_heartbeat_monitor(jcr);
150 jcr->acl_data = (acl_data_t *)malloc(sizeof(acl_data_t));
151 memset((caddr_t)jcr->acl_data, 0, sizeof(acl_data_t));
152 jcr->acl_data->content = get_pool_memory(PM_MESSAGE);
156 jcr->xattr_data = (xattr_data_t *)malloc(sizeof(xattr_data_t));
157 memset((caddr_t)jcr->xattr_data, 0, sizeof(xattr_data_t));
158 jcr->xattr_data->content = get_pool_memory(PM_MESSAGE);
161 /* Subroutine save_file() is called for each file */
162 if (!find_files(jcr, (FF_PKT *)jcr->ff, save_file, plugin_save)) {
163 ok = false; /* error */
164 set_jcr_job_status(jcr, JS_ErrorTerminated);
167 if (have_acl && jcr->acl_data->nr_errors > 0) {
168 Jmsg(jcr, M_ERROR, 0, _("Encountered %ld acl errors while doing backup\n"),
169 jcr->acl_data->nr_errors);
171 if (have_xattr && jcr->xattr_data->nr_errors > 0) {
172 Jmsg(jcr, M_ERROR, 0, _("Encountered %ld xattr errors while doing backup\n"),
173 jcr->xattr_data->nr_errors);
176 accurate_send_deleted_list(jcr); /* send deleted list to SD */
178 stop_heartbeat_monitor(jcr);
180 sd->signal(BNET_EOD); /* end of sending data */
182 if (have_acl && jcr->acl_data) {
183 free_pool_memory(jcr->acl_data->content);
185 jcr->acl_data = NULL;
187 if (have_xattr && jcr->xattr_data) {
188 free_pool_memory(jcr->xattr_data->content);
189 free(jcr->xattr_data);
190 jcr->xattr_data = NULL;
196 if (jcr->compress_buf) {
197 free_pool_memory(jcr->compress_buf);
198 jcr->compress_buf = NULL;
200 if (jcr->pZLIB_compress_workset) {
201 /* Free the zlib stream */
203 deflateEnd((z_stream *)jcr->pZLIB_compress_workset);
205 free (jcr->pZLIB_compress_workset);
206 jcr->pZLIB_compress_workset = NULL;
208 crypto_session_end(jcr);
211 Dmsg1(100, "end blast_data ok=%d\n", ok);
215 static bool crypto_session_start(JCR *jcr)
217 crypto_cipher_t cipher = CRYPTO_CIPHER_AES_128_CBC;
220 * Create encryption session data and a cached, DER-encoded session data
221 * structure. We use a single session key for each backup, so we'll encode
222 * the session data only once.
224 if (jcr->crypto.pki_encrypt) {
227 /* Create per-job session encryption context */
228 jcr->crypto.pki_session = crypto_session_new(cipher, jcr->crypto.pki_recipients);
230 /* Get the session data size */
231 if (!crypto_session_encode(jcr->crypto.pki_session, (uint8_t *)0, &size)) {
232 Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n"));
236 /* Allocate buffer */
237 jcr->crypto.pki_session_encoded = get_memory(size);
239 /* Encode session data */
240 if (!crypto_session_encode(jcr->crypto.pki_session, (uint8_t *)jcr->crypto.pki_session_encoded, &size)) {
241 Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n"));
245 /* ... and store the encoded size */
246 jcr->crypto.pki_session_encoded_size = size;
248 /* Allocate the encryption/decryption buffer */
249 jcr->crypto.crypto_buf = get_memory(CRYPTO_CIPHER_MAX_BLOCK_SIZE);
254 static void crypto_session_end(JCR *jcr)
256 if (jcr->crypto.crypto_buf) {
257 free_pool_memory(jcr->crypto.crypto_buf);
258 jcr->crypto.crypto_buf = NULL;
260 if (jcr->crypto.pki_session) {
261 crypto_session_free(jcr->crypto.pki_session);
263 if (jcr->crypto.pki_session_encoded) {
264 free_pool_memory(jcr->crypto.pki_session_encoded);
265 jcr->crypto.pki_session_encoded = NULL;
269 static bool crypto_session_send(JCR *jcr, BSOCK *sd)
273 /* Send our header */
274 Dmsg2(100, "Send hdr fi=%ld stream=%d\n", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA);
275 sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA);
278 sd->msg = jcr->crypto.pki_session_encoded;
279 sd->msglen = jcr->crypto.pki_session_encoded_size;
280 jcr->JobBytes += sd->msglen;
282 Dmsg1(100, "Send data len=%d\n", sd->msglen);
285 sd->signal(BNET_EOD);
291 * Called here by find() for each file included.
292 * This is a callback. The original is find_files() above.
294 * Send the file and its data to the Storage daemon.
298 * -1 to ignore file/directory (not used here)
300 int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level)
302 bool do_read = false;
303 int stat, data_stream;
305 DIGEST *digest = NULL;
306 DIGEST *signing_digest = NULL;
307 int digest_stream = STREAM_NONE;
308 SIGNATURE *sig = NULL;
309 bool has_file_data = false;
310 // TODO landonf: Allow the user to specify the digest algorithm
312 crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA256;
314 crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA1;
316 BSOCK *sd = jcr->store_bsock;
318 if (job_canceled(jcr)) {
322 jcr->num_files_examined++; /* bump total file count */
324 switch (ff_pkt->type) {
325 case FT_LNKSAVED: /* Hard linked, file already saved */
326 Dmsg2(130, "FT_LNKSAVED hard link: %s => %s\n", ff_pkt->fname, ff_pkt->link);
329 Dmsg1(130, "FT_REGE saving: %s\n", ff_pkt->fname);
330 has_file_data = true;
333 Dmsg1(130, "FT_REG saving: %s\n", ff_pkt->fname);
334 has_file_data = true;
337 Dmsg2(130, "FT_LNK saving: %s -> %s\n", ff_pkt->fname, ff_pkt->link);
340 jcr->num_files_examined--; /* correct file count */
341 return 1; /* not used */
343 Jmsg(jcr, M_INFO, 1, _(" Recursion turned off. Will not descend from %s into %s\n"),
344 ff_pkt->top_fname, ff_pkt->fname);
345 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
348 /* Suppress message for /dev filesystems */
349 if (!is_in_fileset(ff_pkt)) {
350 Jmsg(jcr, M_INFO, 1, _(" %s is a different filesystem. Will not descend from %s into %s\n"),
351 ff_pkt->fname, ff_pkt->top_fname, ff_pkt->fname);
353 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
356 Jmsg(jcr, M_INFO, 1, _(" Disallowed filesystem. Will not descend from %s into %s\n"),
357 ff_pkt->top_fname, ff_pkt->fname);
358 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
361 Jmsg(jcr, M_INFO, 1, _(" Disallowed drive type. Will not descend into %s\n"),
366 Dmsg1(130, "FT_DIREND: %s\n", ff_pkt->link);
369 Dmsg1(130, "FT_SPEC saving: %s\n", ff_pkt->fname);
370 if (S_ISSOCK(ff_pkt->statp.st_mode)) {
371 Jmsg(jcr, M_SKIPPED, 1, _(" Socket file skipped: %s\n"), ff_pkt->fname);
376 Dmsg1(130, "FT_RAW saving: %s\n", ff_pkt->fname);
377 has_file_data = true;
380 Dmsg1(130, "FT_FIFO saving: %s\n", ff_pkt->fname);
384 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not access \"%s\": ERR=%s\n"), ff_pkt->fname,
385 be.bstrerror(ff_pkt->ff_errno));
391 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not follow link \"%s\": ERR=%s\n"),
392 ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
398 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not stat \"%s\": ERR=%s\n"), ff_pkt->fname,
399 be.bstrerror(ff_pkt->ff_errno));
405 Jmsg(jcr, M_SKIPPED, 1, _(" Unchanged file skipped: %s\n"), ff_pkt->fname);
408 Jmsg(jcr, M_NOTSAVED, 0, _(" Archive file not saved: %s\n"), ff_pkt->fname);
412 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not open directory \"%s\": ERR=%s\n"),
413 ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
418 Jmsg(jcr, M_NOTSAVED, 0, _(" Unknown file type %d; not saved: %s\n"),
419 ff_pkt->type, ff_pkt->fname);
424 Dmsg1(130, "bfiled: sending %s to stored\n", ff_pkt->fname);
426 /* Digests and encryption are only useful if there's file data */
429 * Setup for digest handling. If this fails, the digest will be set to NULL
430 * and not used. Note, the digest (file hash) can be any one of the four
433 * The signing digest is a single algorithm depending on
434 * whether or not we have SHA2.
435 * ****FIXME**** the signing algoritm should really be
436 * determined a different way!!!!!! What happens if
437 * sha2 was available during backup but not restore?
439 if (ff_pkt->flags & FO_MD5) {
440 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_MD5);
441 digest_stream = STREAM_MD5_DIGEST;
443 } else if (ff_pkt->flags & FO_SHA1) {
444 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA1);
445 digest_stream = STREAM_SHA1_DIGEST;
447 } else if (ff_pkt->flags & FO_SHA256) {
448 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA256);
449 digest_stream = STREAM_SHA256_DIGEST;
451 } else if (ff_pkt->flags & FO_SHA512) {
452 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA512);
453 digest_stream = STREAM_SHA512_DIGEST;
456 /* Did digest initialization fail? */
457 if (digest_stream != STREAM_NONE && digest == NULL) {
458 Jmsg(jcr, M_WARNING, 0, _("%s digest initialization failed\n"),
459 stream_to_ascii(digest_stream));
463 * Set up signature digest handling. If this fails, the signature digest will be set to
466 // TODO landonf: We should really only calculate the digest once, for both verification and signing.
467 if (jcr->crypto.pki_sign) {
468 signing_digest = crypto_digest_new(jcr, signing_algorithm);
470 /* Full-stop if a failure occurred initializing the signature digest */
471 if (signing_digest == NULL) {
472 Jmsg(jcr, M_NOTSAVED, 0, _("%s signature digest initialization failed\n"),
473 stream_to_ascii(signing_algorithm));
479 /* Enable encryption */
480 if (jcr->crypto.pki_encrypt) {
481 ff_pkt->flags |= FO_ENCRYPT;
485 /* Initialize the file descriptor we use for data and other streams. */
487 if (ff_pkt->flags & FO_PORTABLE) {
488 set_portable_backup(&ff_pkt->bfd); /* disable Win32 BackupRead() */
490 if (ff_pkt->cmd_plugin) {
491 if (!set_cmd_plugin(&ff_pkt->bfd, jcr)) {
494 send_plugin_name(jcr, sd, true); /* signal start of plugin data */
497 /* Send attributes -- must be done after binit() */
498 if (!encode_and_send_attributes(jcr, ff_pkt, data_stream)) {
502 /* Set up the encryption context and send the session data to the SD */
503 if (has_file_data && jcr->crypto.pki_encrypt) {
504 if (!crypto_session_send(jcr, sd)) {
510 * Open any file with data that we intend to save, then save it.
512 * Note, if is_win32_backup, we must open the Directory so that
513 * the BackupRead will save its permissions and ownership streams.
515 if (ff_pkt->type != FT_LNKSAVED && S_ISREG(ff_pkt->statp.st_mode)) {
517 do_read = !is_portable_backup(&ff_pkt->bfd) || ff_pkt->statp.st_size > 0;
519 do_read = ff_pkt->statp.st_size > 0;
521 } else if (ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO ||
522 ff_pkt->type == FT_REPARSE ||
523 (!is_portable_backup(&ff_pkt->bfd) && ff_pkt->type == FT_DIREND)) {
526 if (ff_pkt->cmd_plugin) {
530 Dmsg1(400, "do_read=%d\n", do_read);
534 if (ff_pkt->type == FT_FIFO) {
535 tid = start_thread_timer(jcr, pthread_self(), 60);
539 int noatime = ff_pkt->flags & FO_NOATIME ? O_NOATIME : 0;
540 ff_pkt->bfd.reparse_point = ff_pkt->type == FT_REPARSE;
541 if (bopen(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY | noatime, 0) < 0) {
542 ff_pkt->ff_errno = errno;
544 Jmsg(jcr, M_NOTSAVED, 0, _(" Cannot open \"%s\": ERR=%s.\n"), ff_pkt->fname,
548 stop_thread_timer(tid);
554 stop_thread_timer(tid);
558 stat = send_data(jcr, data_stream, ff_pkt, digest, signing_digest);
560 if (ff_pkt->flags & FO_CHKCHANGES) {
561 has_file_changed(jcr, ff_pkt);
564 bclose(&ff_pkt->bfd);
571 if (have_darwin_os) {
572 /* Regular files can have resource forks and Finder Info */
573 if (ff_pkt->type != FT_LNKSAVED && (S_ISREG(ff_pkt->statp.st_mode) &&
574 ff_pkt->flags & FO_HFSPLUS)) {
575 if (ff_pkt->hfsinfo.rsrclength > 0) {
578 if (!bopen_rsrc(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0) < 0) {
579 ff_pkt->ff_errno = errno;
581 Jmsg(jcr, M_NOTSAVED, -1, _(" Cannot open resource fork for \"%s\": ERR=%s.\n"),
582 ff_pkt->fname, be.bstrerror());
584 if (is_bopen(&ff_pkt->bfd)) {
585 bclose(&ff_pkt->bfd);
589 flags = ff_pkt->flags;
590 ff_pkt->flags &= ~(FO_GZIP|FO_SPARSE);
591 if (flags & FO_ENCRYPT) {
592 rsrc_stream = STREAM_ENCRYPTED_MACOS_FORK_DATA;
594 rsrc_stream = STREAM_MACOS_FORK_DATA;
596 stat = send_data(jcr, rsrc_stream, ff_pkt, digest, signing_digest);
597 ff_pkt->flags = flags;
598 bclose(&ff_pkt->bfd);
604 Dmsg1(300, "Saving Finder Info for \"%s\"\n", ff_pkt->fname);
605 sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_HFSPLUS_ATTRIBUTES);
606 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
607 pm_memcpy(sd->msg, ff_pkt->hfsinfo.fndrinfo, 32);
610 crypto_digest_update(digest, (uint8_t *)sd->msg, sd->msglen);
612 if (signing_digest) {
613 crypto_digest_update(signing_digest, (uint8_t *)sd->msg, sd->msglen);
616 sd->signal(BNET_EOD);
621 * Save ACLs when requested and available for anything not being a symlink and not being a plugin.
624 if (ff_pkt->flags & FO_ACL && ff_pkt->type != FT_LNK && !ff_pkt->cmd_plugin) {
625 switch (build_acl_streams(jcr, ff_pkt)) {
626 case bacl_exit_fatal:
628 case bacl_exit_error:
630 * Non-fatal errors, count them and when the number is under ACL_REPORT_ERR_MAX_PER_JOB
631 * print the error message set by the lower level routine in jcr->errmsg.
633 if (jcr->acl_data->nr_errors < ACL_REPORT_ERR_MAX_PER_JOB) {
634 Jmsg(jcr, M_ERROR, 0, "%s", jcr->errmsg);
636 jcr->acl_data->nr_errors++;
645 * Save Extended Attributes when requested and available for all files not being a plugin.
648 if (ff_pkt->flags & FO_XATTR && !ff_pkt->cmd_plugin) {
649 switch (build_xattr_streams(jcr, ff_pkt)) {
650 case bxattr_exit_fatal:
652 case bxattr_exit_error:
654 * Non-fatal errors, count them and when the number is under XATTR_REPORT_ERR_MAX_PER_JOB
655 * print the error message set by the lower level routine in jcr->errmsg.
657 if (jcr->xattr_data->nr_errors < XATTR_REPORT_ERR_MAX_PER_JOB) {
658 Jmsg(jcr, M_ERROR, 0, "%s", jcr->errmsg);
660 jcr->xattr_data->nr_errors++;
668 /* Terminate the signing digest and send it to the Storage daemon */
669 if (signing_digest) {
672 if ((sig = crypto_sign_new(jcr)) == NULL) {
673 Jmsg(jcr, M_FATAL, 0, _("Failed to allocate memory for crypto signature.\n"));
677 if (!crypto_sign_add_signer(sig, signing_digest, jcr->crypto.pki_keypair)) {
678 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
682 /* Get signature size */
683 if (!crypto_sign_encode(sig, NULL, &size)) {
684 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
688 /* Grow the bsock buffer to fit our message if necessary */
689 if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
690 sd->msg = realloc_pool_memory(sd->msg, size);
693 /* Send our header */
694 sd->fsend("%ld %ld 0", jcr->JobFiles, STREAM_SIGNED_DIGEST);
695 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
697 /* Encode signature data */
698 if (!crypto_sign_encode(sig, (uint8_t *)sd->msg, &size)) {
699 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
705 sd->signal(BNET_EOD); /* end of checksum */
708 /* Terminate any digest and send it to Storage daemon */
712 sd->fsend("%ld %d 0", jcr->JobFiles, digest_stream);
713 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
715 size = CRYPTO_DIGEST_MAX_SIZE;
717 /* Grow the bsock buffer to fit our message if necessary */
718 if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
719 sd->msg = realloc_pool_memory(sd->msg, size);
722 if (!crypto_digest_finalize(digest, (uint8_t *)sd->msg, &size)) {
723 Jmsg(jcr, M_FATAL, 0, _("An error occurred finalizing signing the stream.\n"));
729 sd->signal(BNET_EOD); /* end of checksum */
731 if (ff_pkt->cmd_plugin) {
732 send_plugin_name(jcr, sd, false); /* signal end of plugin data */
736 rtnstat = 1; /* good return */
740 crypto_digest_free(digest);
742 if (signing_digest) {
743 crypto_digest_free(signing_digest);
746 crypto_sign_free(sig);
752 * Send data read from an already open file descriptor.
754 * We return 1 on sucess and 0 on errors.
757 * We use ff_pkt->statp.st_size when FO_SPARSE to know when to stop
759 * Currently this is not a problem as the only other stream, resource forks,
760 * are not handled as sparse files.
762 static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest,
763 DIGEST *signing_digest)
765 BSOCK *sd = jcr->store_bsock;
766 uint64_t fileAddr = 0; /* file address */
768 int32_t rsize = jcr->buf_size; /* read buffer size */
770 CIPHER_CONTEXT *cipher_ctx = NULL; /* Quell bogus uninitialized warnings */
771 const uint8_t *cipher_input;
772 uint32_t cipher_input_len;
773 uint32_t cipher_block_size;
774 uint32_t encrypted_len;
775 #ifdef FD_NO_SEND_TEST
780 rbuf = sd->msg; /* read buffer */
781 wbuf = sd->msg; /* write buffer */
782 cipher_input = (uint8_t *)rbuf; /* encrypt uncompressed data */
784 Dmsg1(300, "Saving data, type=%d\n", ff_pkt->type);
787 uLong compress_len = 0;
788 uLong max_compress_len = 0;
789 const Bytef *cbuf = NULL;
792 if (ff_pkt->flags & FO_GZIP) {
793 if (ff_pkt->flags & FO_SPARSE) {
794 cbuf = (Bytef *)jcr->compress_buf + SPARSE_FADDR_SIZE;
795 max_compress_len = jcr->compress_buf_size - SPARSE_FADDR_SIZE;
797 cbuf = (Bytef *)jcr->compress_buf;
798 max_compress_len = jcr->compress_buf_size; /* set max length */
800 wbuf = jcr->compress_buf; /* compressed output here */
801 cipher_input = (uint8_t *)jcr->compress_buf; /* encrypt compressed data */
804 * Only change zlib parameters if there is no pending operation.
805 * This should never happen as deflatereset is called after each
809 if (((z_stream*)jcr->pZLIB_compress_workset)->total_in == 0) {
810 /* set gzip compression level - must be done per file */
811 if ((zstat=deflateParams((z_stream*)jcr->pZLIB_compress_workset,
812 ff_pkt->GZIP_level, Z_DEFAULT_STRATEGY)) != Z_OK) {
813 Jmsg(jcr, M_FATAL, 0, _("Compression deflateParams error: %d\n"), zstat);
814 set_jcr_job_status(jcr, JS_ErrorTerminated);
820 const uint32_t max_compress_len = 0;
823 if (ff_pkt->flags & FO_ENCRYPT) {
824 if (ff_pkt->flags & FO_SPARSE) {
825 Jmsg0(jcr, M_FATAL, 0, _("Encrypting sparse data not supported.\n"));
828 /* Allocate the cipher context */
829 if ((cipher_ctx = crypto_cipher_new(jcr->crypto.pki_session, true,
830 &cipher_block_size)) == NULL) {
831 /* Shouldn't happen! */
832 Jmsg0(jcr, M_FATAL, 0, _("Failed to initialize encryption context.\n"));
837 * Grow the crypto buffer, if necessary.
838 * crypto_cipher_update() will buffer up to (cipher_block_size - 1).
839 * We grow crypto_buf to the maximum number of blocks that
840 * could be returned for the given read buffer size.
841 * (Using the larger of either rsize or max_compress_len)
843 jcr->crypto.crypto_buf = check_pool_memory_size(jcr->crypto.crypto_buf,
844 (MAX(rsize + (int)sizeof(uint32_t), (int32_t)max_compress_len) +
845 cipher_block_size - 1) / cipher_block_size * cipher_block_size);
847 wbuf = jcr->crypto.crypto_buf; /* Encrypted, possibly compressed output here. */
851 * Send Data header to Storage daemon
852 * <file-index> <stream> <info>
854 if (!sd->fsend("%ld %d 0", jcr->JobFiles, stream)) {
855 if (!job_canceled(jcr)) {
856 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
861 Dmsg1(300, ">stored: datahdr %s\n", sd->msg);
864 * Make space at beginning of buffer for fileAddr because this
865 * same buffer will be used for writing if compression is off.
867 if (ff_pkt->flags & FO_SPARSE) {
868 rbuf += SPARSE_FADDR_SIZE;
869 rsize -= SPARSE_FADDR_SIZE;
870 #ifdef HAVE_FREEBSD_OS
872 * To read FreeBSD partitions, the read size must be
875 rsize = (rsize/512) * 512;
879 /* a RAW device read on win32 only works if the buffer is a multiple of 512 */
881 if (S_ISBLK(ff_pkt->statp.st_mode))
882 rsize = (rsize/512) * 512;
888 while ((sd->msglen=(uint32_t)bread(&ff_pkt->bfd, rbuf, rsize)) > 0) {
890 /* Check for sparse blocks */
891 if (ff_pkt->flags & FO_SPARSE) {
893 bool allZeros = false;
894 if ((sd->msglen == rsize &&
895 fileAddr+sd->msglen < (uint64_t)ff_pkt->statp.st_size) ||
896 ((ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO) &&
897 (uint64_t)ff_pkt->statp.st_size == 0)) {
898 allZeros = is_buf_zero(rbuf, rsize);
901 /* Put file address as first data in buffer */
902 ser_begin(wbuf, SPARSE_FADDR_SIZE);
903 ser_uint64(fileAddr); /* store fileAddr in begin of buffer */
905 fileAddr += sd->msglen; /* update file address */
906 /* Skip block of all zeros */
908 continue; /* skip block of zeros */
912 jcr->ReadBytes += sd->msglen; /* count bytes read */
914 /* Uncompressed cipher input length */
915 cipher_input_len = sd->msglen;
917 /* Update checksum if requested */
919 crypto_digest_update(digest, (uint8_t *)rbuf, sd->msglen);
922 /* Update signing digest if requested */
923 if (signing_digest) {
924 crypto_digest_update(signing_digest, (uint8_t *)rbuf, sd->msglen);
928 /* Do compression if turned on */
929 if (ff_pkt->flags & FO_GZIP && jcr->pZLIB_compress_workset) {
930 Dmsg3(400, "cbuf=0x%x rbuf=0x%x len=%u\n", cbuf, rbuf, sd->msglen);
932 ((z_stream*)jcr->pZLIB_compress_workset)->next_in = (Bytef *)rbuf;
933 ((z_stream*)jcr->pZLIB_compress_workset)->avail_in = sd->msglen;
934 ((z_stream*)jcr->pZLIB_compress_workset)->next_out = (Bytef *)cbuf;
935 ((z_stream*)jcr->pZLIB_compress_workset)->avail_out = max_compress_len;
937 if ((zstat=deflate((z_stream*)jcr->pZLIB_compress_workset, Z_FINISH)) != Z_STREAM_END) {
938 Jmsg(jcr, M_FATAL, 0, _("Compression deflate error: %d\n"), zstat);
939 set_jcr_job_status(jcr, JS_ErrorTerminated);
942 compress_len = ((z_stream*)jcr->pZLIB_compress_workset)->total_out;
943 /* reset zlib stream to be able to begin from scratch again */
944 if ((zstat=deflateReset((z_stream*)jcr->pZLIB_compress_workset)) != Z_OK) {
945 Jmsg(jcr, M_FATAL, 0, _("Compression deflateReset error: %d\n"), zstat);
946 set_jcr_job_status(jcr, JS_ErrorTerminated);
950 Dmsg2(400, "compressed len=%d uncompressed len=%d\n", compress_len,
953 sd->msglen = compress_len; /* set compressed length */
954 cipher_input_len = compress_len;
958 * Note, here we prepend the current record length to the beginning
959 * of the encrypted data. This is because both sparse and compression
960 * restore handling want records returned to them with exactly the
961 * same number of bytes that were processed in the backup handling.
962 * That is, both are block filters rather than a stream. When doing
963 * compression, the compression routines may buffer data, so that for
964 * any one record compressed, when it is decompressed the same size
965 * will not be obtained. Of course, the buffered data eventually comes
966 * out in subsequent crypto_cipher_update() calls or at least
967 * when crypto_cipher_finalize() is called. Unfortunately, this
968 * "feature" of encryption enormously complicates the restore code.
970 if (ff_pkt->flags & FO_ENCRYPT) {
971 uint32_t initial_len = 0;
974 if (ff_pkt->flags & FO_SPARSE) {
975 cipher_input_len += SPARSE_FADDR_SIZE;
978 /* Encrypt the length of the input block */
979 uint8_t packet_len[sizeof(uint32_t)];
981 ser_begin(packet_len, sizeof(uint32_t));
982 ser_uint32(cipher_input_len); /* store data len in begin of buffer */
983 Dmsg1(20, "Encrypt len=%d\n", cipher_input_len);
985 if (!crypto_cipher_update(cipher_ctx, packet_len, sizeof(packet_len),
986 (uint8_t *)jcr->crypto.crypto_buf, &initial_len)) {
987 /* Encryption failed. Shouldn't happen. */
988 Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
992 /* Encrypt the input block */
993 if (crypto_cipher_update(cipher_ctx, cipher_input, cipher_input_len,
994 (uint8_t *)&jcr->crypto.crypto_buf[initial_len], &encrypted_len)) {
995 if ((initial_len + encrypted_len) == 0) {
996 /* No full block of data available, read more data */
999 Dmsg2(400, "encrypted len=%d unencrypted len=%d\n", encrypted_len,
1001 sd->msglen = initial_len + encrypted_len; /* set encrypted length */
1003 /* Encryption failed. Shouldn't happen. */
1004 Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
1009 /* Send the buffer to the Storage daemon */
1010 if (ff_pkt->flags & FO_SPARSE) {
1011 sd->msglen += SPARSE_FADDR_SIZE; /* include fileAddr in size */
1013 sd->msg = wbuf; /* set correct write buffer */
1015 if (!job_canceled(jcr)) {
1016 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1021 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
1023 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
1024 sd->msg = msgsave; /* restore read buffer */
1026 } /* end while read file data */
1028 if (sd->msglen < 0) { /* error */
1030 Jmsg(jcr, M_ERROR, 0, _("Read error on file %s. ERR=%s\n"),
1031 ff_pkt->fname, be.bstrerror(ff_pkt->bfd.berrno));
1032 if (jcr->JobErrors++ > 1000) { /* insanity check */
1033 Jmsg(jcr, M_FATAL, 0, _("Too many errors.\n"));
1035 } else if (ff_pkt->flags & FO_ENCRYPT) {
1037 * For encryption, we must call finalize to push out any
1040 if (!crypto_cipher_finalize(cipher_ctx, (uint8_t *)jcr->crypto.crypto_buf,
1042 /* Padding failed. Shouldn't happen. */
1043 Jmsg(jcr, M_FATAL, 0, _("Encryption padding error\n"));
1047 /* Note, on SSL pre-0.9.7, there is always some output */
1048 if (encrypted_len > 0) {
1049 sd->msglen = encrypted_len; /* set encrypted length */
1050 sd->msg = jcr->crypto.crypto_buf; /* set correct write buffer */
1052 if (!job_canceled(jcr)) {
1053 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1058 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
1059 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
1060 sd->msg = msgsave; /* restore bnet buffer */
1064 if (!sd->signal(BNET_EOD)) { /* indicate end of file data */
1065 if (!job_canceled(jcr)) {
1066 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1072 /* Free the cipher context */
1074 crypto_cipher_free(cipher_ctx);
1079 /* Free the cipher context */
1081 crypto_cipher_free(cipher_ctx);
1084 sd->msg = msgsave; /* restore bnet buffer */
1089 bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream)
1091 BSOCK *sd = jcr->store_bsock;
1092 char attribs[MAXSTRING];
1093 char attribsEx[MAXSTRING];
1096 #ifdef FD_NO_SEND_TEST
1100 Dmsg1(300, "encode_and_send_attrs fname=%s\n", ff_pkt->fname);
1101 /* Find what data stream we will use, then encode the attributes */
1102 if ((data_stream = select_data_stream(ff_pkt)) == STREAM_NONE) {
1103 /* This should not happen */
1104 Jmsg0(jcr, M_FATAL, 0, _("Invalid file flags, no supported data stream type.\n"));
1107 encode_stat(attribs, &ff_pkt->statp, ff_pkt->LinkFI, data_stream);
1109 /* Now possibly extend the attributes */
1110 attr_stream = encode_attribsEx(jcr, attribsEx, ff_pkt);
1112 Dmsg3(300, "File %s\nattribs=%s\nattribsEx=%s\n", ff_pkt->fname, attribs, attribsEx);
1115 jcr->JobFiles++; /* increment number of files sent */
1116 ff_pkt->FileIndex = jcr->JobFiles; /* return FileIndex */
1117 pm_strcpy(jcr->last_fname, ff_pkt->fname);
1121 * Send Attributes header to Storage daemon
1122 * <file-index> <stream> <info>
1124 if (!sd->fsend("%ld %d 0", jcr->JobFiles, attr_stream)) {
1125 if (!job_canceled(jcr)) {
1126 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1131 Dmsg1(300, ">stored: attrhdr %s\n", sd->msg);
1134 * Send file attributes to Storage daemon
1137 * Filename (full path)
1138 * Encoded attributes
1139 * Link name (if type==FT_LNK or FT_LNKSAVED)
1140 * Encoded extended-attributes (for Win32)
1142 * For a directory, link is the same as fname, but with trailing
1143 * slash. For a linked file, link is the link.
1145 if (ff_pkt->type != FT_DELETED) { /* already stripped */
1148 if (ff_pkt->type == FT_LNK || ff_pkt->type == FT_LNKSAVED) {
1149 Dmsg2(300, "Link %s to %s\n", ff_pkt->fname, ff_pkt->link);
1150 stat = sd->fsend("%ld %d %s%c%s%c%s%c%s%c", jcr->JobFiles,
1151 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, ff_pkt->link, 0,
1153 } else if (ff_pkt->type == FT_DIREND || ff_pkt->type == FT_REPARSE) {
1154 /* Here link is the canonical filename (i.e. with trailing slash) */
1155 stat = sd->fsend("%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
1156 ff_pkt->type, ff_pkt->link, 0, attribs, 0, 0, attribsEx, 0);
1158 stat = sd->fsend("%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
1159 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, 0, attribsEx, 0);
1161 if (ff_pkt->type != FT_DELETED) {
1162 unstrip_path(ff_pkt);
1165 Dmsg2(300, ">stored: attr len=%d: %s\n", sd->msglen, sd->msg);
1167 if (!job_canceled(jcr)) {
1168 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1173 sd->signal(BNET_EOD); /* indicate end of attributes data */
1178 * Do in place strip of path
1180 static bool do_strip(int count, char *in)
1186 /* Copy to first path separator -- Win32 might have c: ... */
1187 while (*in && !IsPathSeparator(*in)) {
1191 numsep++; /* one separator seen */
1192 for (stripped=0; stripped<count && *in; stripped++) {
1193 while (*in && !IsPathSeparator(*in)) {
1194 in++; /* skip chars */
1197 numsep++; /* count separators seen */
1198 in++; /* skip separator */
1202 while (*in) { /* copy to end */
1203 if (IsPathSeparator(*in)) {
1209 Dmsg4(500, "stripped=%d count=%d numsep=%d sep>count=%d\n",
1210 stripped, count, numsep, numsep>count);
1211 return stripped==count && numsep>count;
1215 * If requested strip leading components of the path so that we can
1216 * save file as if it came from a subdirectory. This is most useful
1217 * for dealing with snapshots, by removing the snapshot directory, or
1218 * in handling vendor migrations where files have been restored with
1219 * a vendor product into a subdirectory.
1221 void strip_path(FF_PKT *ff_pkt)
1223 if (!(ff_pkt->flags & FO_STRIPPATH) || ff_pkt->strip_path <= 0) {
1224 Dmsg1(200, "No strip for %s\n", ff_pkt->fname);
1227 if (!ff_pkt->fname_save) {
1228 ff_pkt->fname_save = get_pool_memory(PM_FNAME);
1229 ff_pkt->link_save = get_pool_memory(PM_FNAME);
1231 pm_strcpy(ff_pkt->fname_save, ff_pkt->fname);
1232 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1233 pm_strcpy(ff_pkt->link_save, ff_pkt->link);
1234 Dmsg2(500, "strcpy link_save=%d link=%d\n", strlen(ff_pkt->link_save),
1235 strlen(ff_pkt->link));
1236 sm_check(__FILE__, __LINE__, true);
1240 * Strip path. If it doesn't succeed put it back. If
1241 * it does, and there is a different link string,
1242 * attempt to strip the link. If it fails, back them
1244 * Do not strip symlinks.
1245 * I.e. if either stripping fails don't strip anything.
1247 if (!do_strip(ff_pkt->strip_path, ff_pkt->fname)) {
1248 unstrip_path(ff_pkt);
1251 /* Strip links but not symlinks */
1252 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1253 if (!do_strip(ff_pkt->strip_path, ff_pkt->link)) {
1254 unstrip_path(ff_pkt);
1259 Dmsg3(100, "fname=%s stripped=%s link=%s\n", ff_pkt->fname_save, ff_pkt->fname,
1263 void unstrip_path(FF_PKT *ff_pkt)
1265 if (!(ff_pkt->flags & FO_STRIPPATH) || ff_pkt->strip_path <= 0) {
1268 strcpy(ff_pkt->fname, ff_pkt->fname_save);
1269 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1270 Dmsg2(500, "strcpy link=%s link_save=%s\n", ff_pkt->link,
1272 strcpy(ff_pkt->link, ff_pkt->link_save);
1273 Dmsg2(500, "strcpy link=%d link_save=%d\n", strlen(ff_pkt->link),
1274 strlen(ff_pkt->link_save));
1275 sm_check(__FILE__, __LINE__, true);