2 Bacula® - The Network Backup Solution
4 Copyright (C) 2000-2010 Free Software Foundation Europe e.V.
6 The main author of Bacula is Kern Sibbald, with contributions from
7 many others, a complete list can be found in the file AUTHORS.
8 This program is Free Software; you can redistribute it and/or
9 modify it under the terms of version two of the GNU General Public
10 License as published by the Free Software Foundation and included
13 This program is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
23 Bacula® is a registered trademark of Kern Sibbald.
24 The licensor of Bacula is the Free Software Foundation Europe
25 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
26 Switzerland, email:ftf@fsfeurope.org.
29 * Bacula File Daemon backup.c send file attributes and data
30 * to the Storage daemon.
32 * Kern Sibbald, March MM
40 const bool have_darwin_os = true;
42 const bool have_darwin_os = false;
46 const bool have_acl = true;
48 const bool have_acl = false;
51 #if defined(HAVE_XATTR)
52 const bool have_xattr = true;
54 const bool have_xattr = false;
57 /* Forward referenced functions */
58 int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level);
59 static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, DIGEST *signature_digest);
60 bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream);
61 static bool crypto_session_start(JCR *jcr);
62 static void crypto_session_end(JCR *jcr);
63 static bool crypto_session_send(JCR *jcr, BSOCK *sd);
66 * Find all the requested files and send them
67 * to the Storage daemon.
69 * Note, we normally carry on a one-way
70 * conversation from this point on with the SD, simply blasting
71 * data to him. To properly know what is going on, we
72 * also run a "heartbeat" monitor which reads the socket and
73 * reacts accordingly (at the moment it has nothing to do
74 * except echo the heartbeat to the Director).
77 bool blast_data_to_storage_daemon(JCR *jcr, char *addr)
81 // TODO landonf: Allow user to specify encryption algorithm
83 sd = jcr->store_bsock;
85 set_jcr_job_status(jcr, JS_Running);
87 Dmsg1(300, "bfiled: opened data connection %d to stored\n", sd->m_fd);
90 CLIENT *client = (CLIENT *)GetNextRes(R_CLIENT, NULL);
94 buf_size = client->max_network_buffer_size;
96 buf_size = 0; /* use default */
98 if (!sd->set_buffer_size(buf_size, BNET_SETBUF_WRITE)) {
99 set_jcr_job_status(jcr, JS_ErrorTerminated);
100 Jmsg(jcr, M_FATAL, 0, _("Cannot set buffer size FD->SD.\n"));
104 jcr->buf_size = sd->msglen;
106 * Adjust for compression so that output buffer is
107 * 12 bytes + 0.1% larger than input buffer plus 18 bytes.
108 * This gives a bit extra plus room for the sparse addr if any.
109 * Note, we adjust the read size to be smaller so that the
110 * same output buffer can be used without growing it.
112 * The zlib compression workset is initialized here to minimize
113 * the "per file" load. The jcr member is only set, if the init
116 jcr->compress_buf_size = jcr->buf_size + ((jcr->buf_size+999) / 1000) + 30;
117 jcr->compress_buf = get_memory(jcr->compress_buf_size);
120 z_stream *pZlibStream = (z_stream*)malloc(sizeof(z_stream));
122 pZlibStream->zalloc = Z_NULL;
123 pZlibStream->zfree = Z_NULL;
124 pZlibStream->opaque = Z_NULL;
125 pZlibStream->state = Z_NULL;
127 if (deflateInit(pZlibStream, Z_DEFAULT_COMPRESSION) == Z_OK) {
128 jcr->pZLIB_compress_workset = pZlibStream;
135 if (!crypto_session_start(jcr)) {
139 set_find_options((FF_PKT *)jcr->ff, jcr->incremental, jcr->mtime);
141 /* in accurate mode, we overwrite the find_one check function */
143 set_find_changed_function((FF_PKT *)jcr->ff, accurate_check_file);
146 start_heartbeat_monitor(jcr);
149 jcr->acl_data = (acl_data_t *)malloc(sizeof(acl_data_t));
150 memset((caddr_t)jcr->acl_data, 0, sizeof(acl_data_t));
151 jcr->acl_data->content = get_pool_memory(PM_MESSAGE);
155 jcr->xattr_data = (xattr_data_t *)malloc(sizeof(xattr_data_t));
156 memset((caddr_t)jcr->xattr_data, 0, sizeof(xattr_data_t));
157 jcr->xattr_data->content = get_pool_memory(PM_MESSAGE);
160 /* Subroutine save_file() is called for each file */
161 if (!find_files(jcr, (FF_PKT *)jcr->ff, save_file, plugin_save)) {
162 ok = false; /* error */
163 set_jcr_job_status(jcr, JS_ErrorTerminated);
166 if (have_acl && jcr->acl_data->nr_errors > 0) {
167 Jmsg(jcr, M_ERROR, 0, _("Encountered %ld acl errors while doing backup\n"),
168 jcr->acl_data->nr_errors);
170 if (have_xattr && jcr->xattr_data->nr_errors > 0) {
171 Jmsg(jcr, M_ERROR, 0, _("Encountered %ld xattr errors while doing backup\n"),
172 jcr->xattr_data->nr_errors);
175 accurate_finish(jcr); /* send deleted or base file list to SD */
177 stop_heartbeat_monitor(jcr);
179 sd->signal(BNET_EOD); /* end of sending data */
181 if (have_acl && jcr->acl_data) {
182 free_pool_memory(jcr->acl_data->content);
184 jcr->acl_data = NULL;
186 if (have_xattr && jcr->xattr_data) {
187 free_pool_memory(jcr->xattr_data->content);
188 free(jcr->xattr_data);
189 jcr->xattr_data = NULL;
195 if (jcr->compress_buf) {
196 free_pool_memory(jcr->compress_buf);
197 jcr->compress_buf = NULL;
199 if (jcr->pZLIB_compress_workset) {
200 /* Free the zlib stream */
202 deflateEnd((z_stream *)jcr->pZLIB_compress_workset);
204 free (jcr->pZLIB_compress_workset);
205 jcr->pZLIB_compress_workset = NULL;
207 crypto_session_end(jcr);
210 Dmsg1(100, "end blast_data ok=%d\n", ok);
214 static bool crypto_session_start(JCR *jcr)
216 crypto_cipher_t cipher = CRYPTO_CIPHER_AES_128_CBC;
219 * Create encryption session data and a cached, DER-encoded session data
220 * structure. We use a single session key for each backup, so we'll encode
221 * the session data only once.
223 if (jcr->crypto.pki_encrypt) {
226 /* Create per-job session encryption context */
227 jcr->crypto.pki_session = crypto_session_new(cipher, jcr->crypto.pki_recipients);
229 /* Get the session data size */
230 if (!crypto_session_encode(jcr->crypto.pki_session, (uint8_t *)0, &size)) {
231 Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n"));
235 /* Allocate buffer */
236 jcr->crypto.pki_session_encoded = get_memory(size);
238 /* Encode session data */
239 if (!crypto_session_encode(jcr->crypto.pki_session, (uint8_t *)jcr->crypto.pki_session_encoded, &size)) {
240 Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n"));
244 /* ... and store the encoded size */
245 jcr->crypto.pki_session_encoded_size = size;
247 /* Allocate the encryption/decryption buffer */
248 jcr->crypto.crypto_buf = get_memory(CRYPTO_CIPHER_MAX_BLOCK_SIZE);
253 static void crypto_session_end(JCR *jcr)
255 if (jcr->crypto.crypto_buf) {
256 free_pool_memory(jcr->crypto.crypto_buf);
257 jcr->crypto.crypto_buf = NULL;
259 if (jcr->crypto.pki_session) {
260 crypto_session_free(jcr->crypto.pki_session);
262 if (jcr->crypto.pki_session_encoded) {
263 free_pool_memory(jcr->crypto.pki_session_encoded);
264 jcr->crypto.pki_session_encoded = NULL;
268 static bool crypto_session_send(JCR *jcr, BSOCK *sd)
272 /* Send our header */
273 Dmsg2(100, "Send hdr fi=%ld stream=%d\n", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA);
274 sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA);
277 sd->msg = jcr->crypto.pki_session_encoded;
278 sd->msglen = jcr->crypto.pki_session_encoded_size;
279 jcr->JobBytes += sd->msglen;
281 Dmsg1(100, "Send data len=%d\n", sd->msglen);
284 sd->signal(BNET_EOD);
290 * Called here by find() for each file included.
291 * This is a callback. The original is find_files() above.
293 * Send the file and its data to the Storage daemon.
297 * -1 to ignore file/directory (not used here)
299 int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level)
301 bool do_read = false;
302 int stat, data_stream;
304 DIGEST *digest = NULL;
305 DIGEST *signing_digest = NULL;
306 int digest_stream = STREAM_NONE;
307 SIGNATURE *sig = NULL;
308 bool has_file_data = false;
309 // TODO landonf: Allow the user to specify the digest algorithm
311 crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA256;
313 crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA1;
315 BSOCK *sd = jcr->store_bsock;
317 if (job_canceled(jcr)) {
321 jcr->num_files_examined++; /* bump total file count */
323 switch (ff_pkt->type) {
324 case FT_LNKSAVED: /* Hard linked, file already saved */
325 Dmsg2(130, "FT_LNKSAVED hard link: %s => %s\n", ff_pkt->fname, ff_pkt->link);
328 Dmsg1(130, "FT_REGE saving: %s\n", ff_pkt->fname);
329 has_file_data = true;
332 Dmsg1(130, "FT_REG saving: %s\n", ff_pkt->fname);
333 has_file_data = true;
336 Dmsg2(130, "FT_LNK saving: %s -> %s\n", ff_pkt->fname, ff_pkt->link);
339 jcr->num_files_examined--; /* correct file count */
340 return 1; /* not used */
342 Jmsg(jcr, M_INFO, 1, _(" Recursion turned off. Will not descend from %s into %s\n"),
343 ff_pkt->top_fname, ff_pkt->fname);
344 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
347 /* Suppress message for /dev filesystems */
348 if (!is_in_fileset(ff_pkt)) {
349 Jmsg(jcr, M_INFO, 1, _(" %s is a different filesystem. Will not descend from %s into %s\n"),
350 ff_pkt->fname, ff_pkt->top_fname, ff_pkt->fname);
352 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
355 Jmsg(jcr, M_INFO, 1, _(" Disallowed filesystem. Will not descend from %s into %s\n"),
356 ff_pkt->top_fname, ff_pkt->fname);
357 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
360 Jmsg(jcr, M_INFO, 1, _(" Disallowed drive type. Will not descend into %s\n"),
365 Dmsg1(130, "FT_DIREND: %s\n", ff_pkt->link);
368 Dmsg1(130, "FT_SPEC saving: %s\n", ff_pkt->fname);
369 if (S_ISSOCK(ff_pkt->statp.st_mode)) {
370 Jmsg(jcr, M_SKIPPED, 1, _(" Socket file skipped: %s\n"), ff_pkt->fname);
375 Dmsg1(130, "FT_RAW saving: %s\n", ff_pkt->fname);
376 has_file_data = true;
379 Dmsg1(130, "FT_FIFO saving: %s\n", ff_pkt->fname);
383 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not access \"%s\": ERR=%s\n"), ff_pkt->fname,
384 be.bstrerror(ff_pkt->ff_errno));
390 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not follow link \"%s\": ERR=%s\n"),
391 ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
397 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not stat \"%s\": ERR=%s\n"), ff_pkt->fname,
398 be.bstrerror(ff_pkt->ff_errno));
404 Jmsg(jcr, M_SKIPPED, 1, _(" Unchanged file skipped: %s\n"), ff_pkt->fname);
407 Jmsg(jcr, M_NOTSAVED, 0, _(" Archive file not saved: %s\n"), ff_pkt->fname);
411 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not open directory \"%s\": ERR=%s\n"),
412 ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
417 Jmsg(jcr, M_NOTSAVED, 0, _(" Unknown file type %d; not saved: %s\n"),
418 ff_pkt->type, ff_pkt->fname);
423 Dmsg1(130, "bfiled: sending %s to stored\n", ff_pkt->fname);
425 /* Digests and encryption are only useful if there's file data */
428 * Setup for digest handling. If this fails, the digest will be set to NULL
429 * and not used. Note, the digest (file hash) can be any one of the four
432 * The signing digest is a single algorithm depending on
433 * whether or not we have SHA2.
434 * ****FIXME**** the signing algoritm should really be
435 * determined a different way!!!!!! What happens if
436 * sha2 was available during backup but not restore?
438 if (ff_pkt->flags & FO_MD5) {
439 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_MD5);
440 digest_stream = STREAM_MD5_DIGEST;
442 } else if (ff_pkt->flags & FO_SHA1) {
443 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA1);
444 digest_stream = STREAM_SHA1_DIGEST;
446 } else if (ff_pkt->flags & FO_SHA256) {
447 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA256);
448 digest_stream = STREAM_SHA256_DIGEST;
450 } else if (ff_pkt->flags & FO_SHA512) {
451 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA512);
452 digest_stream = STREAM_SHA512_DIGEST;
455 /* Did digest initialization fail? */
456 if (digest_stream != STREAM_NONE && digest == NULL) {
457 Jmsg(jcr, M_WARNING, 0, _("%s digest initialization failed\n"),
458 stream_to_ascii(digest_stream));
462 * Set up signature digest handling. If this fails, the signature digest
463 * will be set to NULL and not used.
465 /* TODO landonf: We should really only calculate the digest once, for
466 * both verification and signing.
468 if (jcr->crypto.pki_sign) {
469 signing_digest = crypto_digest_new(jcr, signing_algorithm);
471 /* Full-stop if a failure occurred initializing the signature digest */
472 if (signing_digest == NULL) {
473 Jmsg(jcr, M_NOTSAVED, 0, _("%s signature digest initialization failed\n"),
474 stream_to_ascii(signing_algorithm));
480 /* Enable encryption */
481 if (jcr->crypto.pki_encrypt) {
482 ff_pkt->flags |= FO_ENCRYPT;
486 /* Initialize the file descriptor we use for data and other streams. */
488 if (ff_pkt->flags & FO_PORTABLE) {
489 set_portable_backup(&ff_pkt->bfd); /* disable Win32 BackupRead() */
491 if (ff_pkt->cmd_plugin) {
492 if (!set_cmd_plugin(&ff_pkt->bfd, jcr)) {
495 send_plugin_name(jcr, sd, true); /* signal start of plugin data */
498 /* Send attributes -- must be done after binit() */
499 if (!encode_and_send_attributes(jcr, ff_pkt, data_stream)) {
503 /* Set up the encryption context and send the session data to the SD */
504 if (has_file_data && jcr->crypto.pki_encrypt) {
505 if (!crypto_session_send(jcr, sd)) {
511 * Open any file with data that we intend to save, then save it.
513 * Note, if is_win32_backup, we must open the Directory so that
514 * the BackupRead will save its permissions and ownership streams.
516 if (ff_pkt->type != FT_LNKSAVED && S_ISREG(ff_pkt->statp.st_mode)) {
518 do_read = !is_portable_backup(&ff_pkt->bfd) || ff_pkt->statp.st_size > 0;
520 do_read = ff_pkt->statp.st_size > 0;
522 } else if (ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO ||
523 ff_pkt->type == FT_REPARSE ||
524 (!is_portable_backup(&ff_pkt->bfd) && ff_pkt->type == FT_DIREND)) {
527 if (ff_pkt->cmd_plugin) {
531 Dmsg1(400, "do_read=%d\n", do_read);
535 if (ff_pkt->type == FT_FIFO) {
536 tid = start_thread_timer(jcr, pthread_self(), 60);
540 int noatime = ff_pkt->flags & FO_NOATIME ? O_NOATIME : 0;
541 ff_pkt->bfd.reparse_point = ff_pkt->type == FT_REPARSE;
542 if (bopen(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY | noatime, 0) < 0) {
543 ff_pkt->ff_errno = errno;
545 Jmsg(jcr, M_NOTSAVED, 0, _(" Cannot open \"%s\": ERR=%s.\n"), ff_pkt->fname,
549 stop_thread_timer(tid);
555 stop_thread_timer(tid);
559 stat = send_data(jcr, data_stream, ff_pkt, digest, signing_digest);
561 if (ff_pkt->flags & FO_CHKCHANGES) {
562 has_file_changed(jcr, ff_pkt);
565 bclose(&ff_pkt->bfd);
572 if (have_darwin_os) {
573 /* Regular files can have resource forks and Finder Info */
574 if (ff_pkt->type != FT_LNKSAVED && (S_ISREG(ff_pkt->statp.st_mode) &&
575 ff_pkt->flags & FO_HFSPLUS)) {
576 if (ff_pkt->hfsinfo.rsrclength > 0) {
579 if (!bopen_rsrc(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0) < 0) {
580 ff_pkt->ff_errno = errno;
582 Jmsg(jcr, M_NOTSAVED, -1, _(" Cannot open resource fork for \"%s\": ERR=%s.\n"),
583 ff_pkt->fname, be.bstrerror());
585 if (is_bopen(&ff_pkt->bfd)) {
586 bclose(&ff_pkt->bfd);
590 flags = ff_pkt->flags;
591 ff_pkt->flags &= ~(FO_GZIP|FO_SPARSE);
592 if (flags & FO_ENCRYPT) {
593 rsrc_stream = STREAM_ENCRYPTED_MACOS_FORK_DATA;
595 rsrc_stream = STREAM_MACOS_FORK_DATA;
597 stat = send_data(jcr, rsrc_stream, ff_pkt, digest, signing_digest);
598 ff_pkt->flags = flags;
599 bclose(&ff_pkt->bfd);
605 Dmsg1(300, "Saving Finder Info for \"%s\"\n", ff_pkt->fname);
606 sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_HFSPLUS_ATTRIBUTES);
607 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
608 pm_memcpy(sd->msg, ff_pkt->hfsinfo.fndrinfo, 32);
611 crypto_digest_update(digest, (uint8_t *)sd->msg, sd->msglen);
613 if (signing_digest) {
614 crypto_digest_update(signing_digest, (uint8_t *)sd->msg, sd->msglen);
617 sd->signal(BNET_EOD);
622 * Save ACLs when requested and available for anything not being a symlink and not being a plugin.
625 if (ff_pkt->flags & FO_ACL && ff_pkt->type != FT_LNK && !ff_pkt->cmd_plugin) {
626 switch (build_acl_streams(jcr, ff_pkt)) {
627 case bacl_exit_fatal:
629 case bacl_exit_error:
631 * Non-fatal errors, count them and when the number is under ACL_REPORT_ERR_MAX_PER_JOB
632 * print the error message set by the lower level routine in jcr->errmsg.
634 if (jcr->acl_data->nr_errors < ACL_REPORT_ERR_MAX_PER_JOB) {
635 Jmsg(jcr, M_ERROR, 0, "%s", jcr->errmsg);
637 jcr->acl_data->nr_errors++;
646 * Save Extended Attributes when requested and available for all files not being a plugin.
649 if (ff_pkt->flags & FO_XATTR && !ff_pkt->cmd_plugin) {
650 switch (build_xattr_streams(jcr, ff_pkt)) {
651 case bxattr_exit_fatal:
653 case bxattr_exit_error:
655 * Non-fatal errors, count them and when the number is under XATTR_REPORT_ERR_MAX_PER_JOB
656 * print the error message set by the lower level routine in jcr->errmsg.
658 if (jcr->xattr_data->nr_errors < XATTR_REPORT_ERR_MAX_PER_JOB) {
659 Jmsg(jcr, M_ERROR, 0, "%s", jcr->errmsg);
661 jcr->xattr_data->nr_errors++;
669 /* Terminate the signing digest and send it to the Storage daemon */
670 if (signing_digest) {
673 if ((sig = crypto_sign_new(jcr)) == NULL) {
674 Jmsg(jcr, M_FATAL, 0, _("Failed to allocate memory for crypto signature.\n"));
678 if (!crypto_sign_add_signer(sig, signing_digest, jcr->crypto.pki_keypair)) {
679 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
683 /* Get signature size */
684 if (!crypto_sign_encode(sig, NULL, &size)) {
685 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
689 /* Grow the bsock buffer to fit our message if necessary */
690 if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
691 sd->msg = realloc_pool_memory(sd->msg, size);
694 /* Send our header */
695 sd->fsend("%ld %ld 0", jcr->JobFiles, STREAM_SIGNED_DIGEST);
696 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
698 /* Encode signature data */
699 if (!crypto_sign_encode(sig, (uint8_t *)sd->msg, &size)) {
700 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
706 sd->signal(BNET_EOD); /* end of checksum */
709 /* Terminate any digest and send it to Storage daemon */
713 sd->fsend("%ld %d 0", jcr->JobFiles, digest_stream);
714 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
716 size = CRYPTO_DIGEST_MAX_SIZE;
718 /* Grow the bsock buffer to fit our message if necessary */
719 if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
720 sd->msg = realloc_pool_memory(sd->msg, size);
723 if (!crypto_digest_finalize(digest, (uint8_t *)sd->msg, &size)) {
724 Jmsg(jcr, M_FATAL, 0, _("An error occurred finalizing signing the stream.\n"));
730 sd->signal(BNET_EOD); /* end of checksum */
732 if (ff_pkt->cmd_plugin) {
733 send_plugin_name(jcr, sd, false); /* signal end of plugin data */
737 rtnstat = 1; /* good return */
741 crypto_digest_free(digest);
743 if (signing_digest) {
744 crypto_digest_free(signing_digest);
747 crypto_sign_free(sig);
753 * Send data read from an already open file descriptor.
755 * We return 1 on sucess and 0 on errors.
758 * We use ff_pkt->statp.st_size when FO_SPARSE to know when to stop
760 * Currently this is not a problem as the only other stream, resource forks,
761 * are not handled as sparse files.
763 static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest,
764 DIGEST *signing_digest)
766 BSOCK *sd = jcr->store_bsock;
767 uint64_t fileAddr = 0; /* file address */
769 int32_t rsize = jcr->buf_size; /* read buffer size */
771 CIPHER_CONTEXT *cipher_ctx = NULL; /* Quell bogus uninitialized warnings */
772 const uint8_t *cipher_input;
773 uint32_t cipher_input_len;
774 uint32_t cipher_block_size;
775 uint32_t encrypted_len;
776 #ifdef FD_NO_SEND_TEST
781 rbuf = sd->msg; /* read buffer */
782 wbuf = sd->msg; /* write buffer */
783 cipher_input = (uint8_t *)rbuf; /* encrypt uncompressed data */
785 Dmsg1(300, "Saving data, type=%d\n", ff_pkt->type);
788 uLong compress_len = 0;
789 uLong max_compress_len = 0;
790 const Bytef *cbuf = NULL;
793 if (ff_pkt->flags & FO_GZIP) {
794 if (ff_pkt->flags & FO_SPARSE) {
795 cbuf = (Bytef *)jcr->compress_buf + SPARSE_FADDR_SIZE;
796 max_compress_len = jcr->compress_buf_size - SPARSE_FADDR_SIZE;
798 cbuf = (Bytef *)jcr->compress_buf;
799 max_compress_len = jcr->compress_buf_size; /* set max length */
801 wbuf = jcr->compress_buf; /* compressed output here */
802 cipher_input = (uint8_t *)jcr->compress_buf; /* encrypt compressed data */
805 * Only change zlib parameters if there is no pending operation.
806 * This should never happen as deflatereset is called after each
810 if (((z_stream*)jcr->pZLIB_compress_workset)->total_in == 0) {
811 /* set gzip compression level - must be done per file */
812 if ((zstat=deflateParams((z_stream*)jcr->pZLIB_compress_workset,
813 ff_pkt->GZIP_level, Z_DEFAULT_STRATEGY)) != Z_OK) {
814 Jmsg(jcr, M_FATAL, 0, _("Compression deflateParams error: %d\n"), zstat);
815 set_jcr_job_status(jcr, JS_ErrorTerminated);
821 const uint32_t max_compress_len = 0;
824 if (ff_pkt->flags & FO_ENCRYPT) {
825 if (ff_pkt->flags & FO_SPARSE) {
826 Jmsg0(jcr, M_FATAL, 0, _("Encrypting sparse data not supported.\n"));
829 /* Allocate the cipher context */
830 if ((cipher_ctx = crypto_cipher_new(jcr->crypto.pki_session, true,
831 &cipher_block_size)) == NULL) {
832 /* Shouldn't happen! */
833 Jmsg0(jcr, M_FATAL, 0, _("Failed to initialize encryption context.\n"));
838 * Grow the crypto buffer, if necessary.
839 * crypto_cipher_update() will buffer up to (cipher_block_size - 1).
840 * We grow crypto_buf to the maximum number of blocks that
841 * could be returned for the given read buffer size.
842 * (Using the larger of either rsize or max_compress_len)
844 jcr->crypto.crypto_buf = check_pool_memory_size(jcr->crypto.crypto_buf,
845 (MAX(rsize + (int)sizeof(uint32_t), (int32_t)max_compress_len) +
846 cipher_block_size - 1) / cipher_block_size * cipher_block_size);
848 wbuf = jcr->crypto.crypto_buf; /* Encrypted, possibly compressed output here. */
852 * Send Data header to Storage daemon
853 * <file-index> <stream> <info>
855 if (!sd->fsend("%ld %d 0", jcr->JobFiles, stream)) {
856 if (!job_canceled(jcr)) {
857 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
862 Dmsg1(300, ">stored: datahdr %s\n", sd->msg);
865 * Make space at beginning of buffer for fileAddr because this
866 * same buffer will be used for writing if compression is off.
868 if (ff_pkt->flags & FO_SPARSE) {
869 rbuf += SPARSE_FADDR_SIZE;
870 rsize -= SPARSE_FADDR_SIZE;
871 #ifdef HAVE_FREEBSD_OS
873 * To read FreeBSD partitions, the read size must be
876 rsize = (rsize/512) * 512;
880 /* a RAW device read on win32 only works if the buffer is a multiple of 512 */
882 if (S_ISBLK(ff_pkt->statp.st_mode))
883 rsize = (rsize/512) * 512;
889 while ((sd->msglen=(uint32_t)bread(&ff_pkt->bfd, rbuf, rsize)) > 0) {
891 /* Check for sparse blocks */
892 if (ff_pkt->flags & FO_SPARSE) {
894 bool allZeros = false;
895 if ((sd->msglen == rsize &&
896 fileAddr+sd->msglen < (uint64_t)ff_pkt->statp.st_size) ||
897 ((ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO) &&
898 (uint64_t)ff_pkt->statp.st_size == 0)) {
899 allZeros = is_buf_zero(rbuf, rsize);
902 /* Put file address as first data in buffer */
903 ser_begin(wbuf, SPARSE_FADDR_SIZE);
904 ser_uint64(fileAddr); /* store fileAddr in begin of buffer */
906 fileAddr += sd->msglen; /* update file address */
907 /* Skip block of all zeros */
909 continue; /* skip block of zeros */
913 jcr->ReadBytes += sd->msglen; /* count bytes read */
915 /* Uncompressed cipher input length */
916 cipher_input_len = sd->msglen;
918 /* Update checksum if requested */
920 crypto_digest_update(digest, (uint8_t *)rbuf, sd->msglen);
923 /* Update signing digest if requested */
924 if (signing_digest) {
925 crypto_digest_update(signing_digest, (uint8_t *)rbuf, sd->msglen);
929 /* Do compression if turned on */
930 if (ff_pkt->flags & FO_GZIP && jcr->pZLIB_compress_workset) {
931 Dmsg3(400, "cbuf=0x%x rbuf=0x%x len=%u\n", cbuf, rbuf, sd->msglen);
933 ((z_stream*)jcr->pZLIB_compress_workset)->next_in = (Bytef *)rbuf;
934 ((z_stream*)jcr->pZLIB_compress_workset)->avail_in = sd->msglen;
935 ((z_stream*)jcr->pZLIB_compress_workset)->next_out = (Bytef *)cbuf;
936 ((z_stream*)jcr->pZLIB_compress_workset)->avail_out = max_compress_len;
938 if ((zstat=deflate((z_stream*)jcr->pZLIB_compress_workset, Z_FINISH)) != Z_STREAM_END) {
939 Jmsg(jcr, M_FATAL, 0, _("Compression deflate error: %d\n"), zstat);
940 set_jcr_job_status(jcr, JS_ErrorTerminated);
943 compress_len = ((z_stream*)jcr->pZLIB_compress_workset)->total_out;
944 /* reset zlib stream to be able to begin from scratch again */
945 if ((zstat=deflateReset((z_stream*)jcr->pZLIB_compress_workset)) != Z_OK) {
946 Jmsg(jcr, M_FATAL, 0, _("Compression deflateReset error: %d\n"), zstat);
947 set_jcr_job_status(jcr, JS_ErrorTerminated);
951 Dmsg2(400, "compressed len=%d uncompressed len=%d\n", compress_len,
954 sd->msglen = compress_len; /* set compressed length */
955 cipher_input_len = compress_len;
959 * Note, here we prepend the current record length to the beginning
960 * of the encrypted data. This is because both sparse and compression
961 * restore handling want records returned to them with exactly the
962 * same number of bytes that were processed in the backup handling.
963 * That is, both are block filters rather than a stream. When doing
964 * compression, the compression routines may buffer data, so that for
965 * any one record compressed, when it is decompressed the same size
966 * will not be obtained. Of course, the buffered data eventually comes
967 * out in subsequent crypto_cipher_update() calls or at least
968 * when crypto_cipher_finalize() is called. Unfortunately, this
969 * "feature" of encryption enormously complicates the restore code.
971 if (ff_pkt->flags & FO_ENCRYPT) {
972 uint32_t initial_len = 0;
975 if (ff_pkt->flags & FO_SPARSE) {
976 cipher_input_len += SPARSE_FADDR_SIZE;
979 /* Encrypt the length of the input block */
980 uint8_t packet_len[sizeof(uint32_t)];
982 ser_begin(packet_len, sizeof(uint32_t));
983 ser_uint32(cipher_input_len); /* store data len in begin of buffer */
984 Dmsg1(20, "Encrypt len=%d\n", cipher_input_len);
986 if (!crypto_cipher_update(cipher_ctx, packet_len, sizeof(packet_len),
987 (uint8_t *)jcr->crypto.crypto_buf, &initial_len)) {
988 /* Encryption failed. Shouldn't happen. */
989 Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
993 /* Encrypt the input block */
994 if (crypto_cipher_update(cipher_ctx, cipher_input, cipher_input_len,
995 (uint8_t *)&jcr->crypto.crypto_buf[initial_len], &encrypted_len)) {
996 if ((initial_len + encrypted_len) == 0) {
997 /* No full block of data available, read more data */
1000 Dmsg2(400, "encrypted len=%d unencrypted len=%d\n", encrypted_len,
1002 sd->msglen = initial_len + encrypted_len; /* set encrypted length */
1004 /* Encryption failed. Shouldn't happen. */
1005 Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
1010 /* Send the buffer to the Storage daemon */
1011 if (ff_pkt->flags & FO_SPARSE) {
1012 sd->msglen += SPARSE_FADDR_SIZE; /* include fileAddr in size */
1014 sd->msg = wbuf; /* set correct write buffer */
1016 if (!job_canceled(jcr)) {
1017 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1022 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
1024 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
1025 sd->msg = msgsave; /* restore read buffer */
1027 } /* end while read file data */
1029 if (sd->msglen < 0) { /* error */
1031 Jmsg(jcr, M_ERROR, 0, _("Read error on file %s. ERR=%s\n"),
1032 ff_pkt->fname, be.bstrerror(ff_pkt->bfd.berrno));
1033 if (jcr->JobErrors++ > 1000) { /* insanity check */
1034 Jmsg(jcr, M_FATAL, 0, _("Too many errors. JobErrors=%d.\n"), jcr->JobErrors);
1036 } else if (ff_pkt->flags & FO_ENCRYPT) {
1038 * For encryption, we must call finalize to push out any
1041 if (!crypto_cipher_finalize(cipher_ctx, (uint8_t *)jcr->crypto.crypto_buf,
1043 /* Padding failed. Shouldn't happen. */
1044 Jmsg(jcr, M_FATAL, 0, _("Encryption padding error\n"));
1048 /* Note, on SSL pre-0.9.7, there is always some output */
1049 if (encrypted_len > 0) {
1050 sd->msglen = encrypted_len; /* set encrypted length */
1051 sd->msg = jcr->crypto.crypto_buf; /* set correct write buffer */
1053 if (!job_canceled(jcr)) {
1054 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1059 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
1060 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
1061 sd->msg = msgsave; /* restore bnet buffer */
1065 if (!sd->signal(BNET_EOD)) { /* indicate end of file data */
1066 if (!job_canceled(jcr)) {
1067 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1073 /* Free the cipher context */
1075 crypto_cipher_free(cipher_ctx);
1080 /* Free the cipher context */
1082 crypto_cipher_free(cipher_ctx);
1085 sd->msg = msgsave; /* restore bnet buffer */
1090 bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream)
1092 BSOCK *sd = jcr->store_bsock;
1093 char attribs[MAXSTRING];
1094 char attribsEx[MAXSTRING];
1097 #ifdef FD_NO_SEND_TEST
1101 Dmsg1(300, "encode_and_send_attrs fname=%s\n", ff_pkt->fname);
1102 /* Find what data stream we will use, then encode the attributes */
1103 if ((data_stream = select_data_stream(ff_pkt)) == STREAM_NONE) {
1104 /* This should not happen */
1105 Jmsg0(jcr, M_FATAL, 0, _("Invalid file flags, no supported data stream type.\n"));
1108 encode_stat(attribs, &ff_pkt->statp, ff_pkt->LinkFI, data_stream);
1110 /* Now possibly extend the attributes */
1111 attr_stream = encode_attribsEx(jcr, attribsEx, ff_pkt);
1113 Dmsg3(300, "File %s\nattribs=%s\nattribsEx=%s\n", ff_pkt->fname, attribs, attribsEx);
1116 jcr->JobFiles++; /* increment number of files sent */
1117 ff_pkt->FileIndex = jcr->JobFiles; /* return FileIndex */
1118 pm_strcpy(jcr->last_fname, ff_pkt->fname);
1122 * Send Attributes header to Storage daemon
1123 * <file-index> <stream> <info>
1125 if (!sd->fsend("%ld %d 0", jcr->JobFiles, attr_stream)) {
1126 if (!job_canceled(jcr)) {
1127 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1132 Dmsg1(300, ">stored: attrhdr %s\n", sd->msg);
1135 * Send file attributes to Storage daemon
1138 * Filename (full path)
1139 * Encoded attributes
1140 * Link name (if type==FT_LNK or FT_LNKSAVED)
1141 * Encoded extended-attributes (for Win32)
1143 * For a directory, link is the same as fname, but with trailing
1144 * slash. For a linked file, link is the link.
1146 if (ff_pkt->type != FT_DELETED) { /* already stripped */
1149 if (ff_pkt->type == FT_LNK || ff_pkt->type == FT_LNKSAVED) {
1150 Dmsg2(300, "Link %s to %s\n", ff_pkt->fname, ff_pkt->link);
1151 stat = sd->fsend("%ld %d %s%c%s%c%s%c%s%c", jcr->JobFiles,
1152 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, ff_pkt->link, 0,
1154 } else if (ff_pkt->type == FT_DIREND || ff_pkt->type == FT_REPARSE) {
1155 /* Here link is the canonical filename (i.e. with trailing slash) */
1156 stat = sd->fsend("%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
1157 ff_pkt->type, ff_pkt->link, 0, attribs, 0, 0, attribsEx, 0);
1159 stat = sd->fsend("%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
1160 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, 0, attribsEx, 0);
1162 if (ff_pkt->type != FT_DELETED) {
1163 unstrip_path(ff_pkt);
1166 Dmsg2(300, ">stored: attr len=%d: %s\n", sd->msglen, sd->msg);
1168 if (!job_canceled(jcr)) {
1169 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1174 sd->signal(BNET_EOD); /* indicate end of attributes data */
1179 * Do in place strip of path
1181 static bool do_strip(int count, char *in)
1187 /* Copy to first path separator -- Win32 might have c: ... */
1188 while (*in && !IsPathSeparator(*in)) {
1192 numsep++; /* one separator seen */
1193 for (stripped=0; stripped<count && *in; stripped++) {
1194 while (*in && !IsPathSeparator(*in)) {
1195 in++; /* skip chars */
1198 numsep++; /* count separators seen */
1199 in++; /* skip separator */
1203 while (*in) { /* copy to end */
1204 if (IsPathSeparator(*in)) {
1210 Dmsg4(500, "stripped=%d count=%d numsep=%d sep>count=%d\n",
1211 stripped, count, numsep, numsep>count);
1212 return stripped==count && numsep>count;
1216 * If requested strip leading components of the path so that we can
1217 * save file as if it came from a subdirectory. This is most useful
1218 * for dealing with snapshots, by removing the snapshot directory, or
1219 * in handling vendor migrations where files have been restored with
1220 * a vendor product into a subdirectory.
1222 void strip_path(FF_PKT *ff_pkt)
1224 if (!(ff_pkt->flags & FO_STRIPPATH) || ff_pkt->strip_path <= 0) {
1225 Dmsg1(200, "No strip for %s\n", ff_pkt->fname);
1228 if (!ff_pkt->fname_save) {
1229 ff_pkt->fname_save = get_pool_memory(PM_FNAME);
1230 ff_pkt->link_save = get_pool_memory(PM_FNAME);
1232 pm_strcpy(ff_pkt->fname_save, ff_pkt->fname);
1233 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1234 pm_strcpy(ff_pkt->link_save, ff_pkt->link);
1235 Dmsg2(500, "strcpy link_save=%d link=%d\n", strlen(ff_pkt->link_save),
1236 strlen(ff_pkt->link));
1237 sm_check(__FILE__, __LINE__, true);
1241 * Strip path. If it doesn't succeed put it back. If
1242 * it does, and there is a different link string,
1243 * attempt to strip the link. If it fails, back them
1245 * Do not strip symlinks.
1246 * I.e. if either stripping fails don't strip anything.
1248 if (!do_strip(ff_pkt->strip_path, ff_pkt->fname)) {
1249 unstrip_path(ff_pkt);
1252 /** Strip links but not symlinks */
1253 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1254 if (!do_strip(ff_pkt->strip_path, ff_pkt->link)) {
1255 unstrip_path(ff_pkt);
1260 Dmsg3(100, "fname=%s stripped=%s link=%s\n", ff_pkt->fname_save, ff_pkt->fname,
1264 void unstrip_path(FF_PKT *ff_pkt)
1266 if (!(ff_pkt->flags & FO_STRIPPATH) || ff_pkt->strip_path <= 0) {
1269 strcpy(ff_pkt->fname, ff_pkt->fname_save);
1270 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1271 Dmsg2(500, "strcpy link=%s link_save=%s\n", ff_pkt->link,
1273 strcpy(ff_pkt->link, ff_pkt->link_save);
1274 Dmsg2(500, "strcpy link=%d link_save=%d\n", strlen(ff_pkt->link),
1275 strlen(ff_pkt->link_save));
1276 sm_check(__FILE__, __LINE__, true);