2 Bacula® - The Network Backup Solution
4 Copyright (C) 2000-2009 Free Software Foundation Europe e.V.
6 The main author of Bacula is Kern Sibbald, with contributions from
7 many others, a complete list can be found in the file AUTHORS.
8 This program is Free Software; you can redistribute it and/or
9 modify it under the terms of version two of the GNU General Public
10 License as published by the Free Software Foundation and included
13 This program is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
23 Bacula® is a registered trademark of Kern Sibbald.
24 The licensor of Bacula is the Free Software Foundation Europe
25 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
26 Switzerland, email:ftf@fsfeurope.org.
29 * Bacula File Daemon backup.c send file attributes and data
30 * to the Storage daemon.
32 * Kern Sibbald, March MM
42 const bool have_darwin_os = true;
44 const bool have_darwin_os = false;
48 const bool have_acl = true;
50 const bool have_acl = false;
53 #if defined(HAVE_XATTR)
54 const bool have_xattr = true;
56 const bool have_xattr = false;
59 /* Forward referenced functions */
60 int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level);
61 static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, DIGEST *signature_digest);
62 bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream);
63 static bool crypto_session_start(JCR *jcr);
64 static void crypto_session_end(JCR *jcr);
65 static bool crypto_session_send(JCR *jcr, BSOCK *sd);
68 * Find all the requested files and send them
69 * to the Storage daemon.
71 * Note, we normally carry on a one-way
72 * conversation from this point on with the SD, simply blasting
73 * data to him. To properly know what is going on, we
74 * also run a "heartbeat" monitor which reads the socket and
75 * reacts accordingly (at the moment it has nothing to do
76 * except echo the heartbeat to the Director).
79 bool blast_data_to_storage_daemon(JCR *jcr, char *addr)
83 // TODO landonf: Allow user to specify encryption algorithm
85 sd = jcr->store_bsock;
87 set_jcr_job_status(jcr, JS_Running);
89 Dmsg1(300, "bfiled: opened data connection %d to stored\n", sd->m_fd);
92 CLIENT *client = (CLIENT *)GetNextRes(R_CLIENT, NULL);
96 buf_size = client->max_network_buffer_size;
98 buf_size = 0; /* use default */
100 if (!sd->set_buffer_size(buf_size, BNET_SETBUF_WRITE)) {
101 set_jcr_job_status(jcr, JS_ErrorTerminated);
102 Jmsg(jcr, M_FATAL, 0, _("Cannot set buffer size FD->SD.\n"));
106 jcr->buf_size = sd->msglen;
107 /* Adjust for compression so that output buffer is
108 * 12 bytes + 0.1% larger than input buffer plus 18 bytes.
109 * This gives a bit extra plus room for the sparse addr if any.
110 * Note, we adjust the read size to be smaller so that the
111 * same output buffer can be used without growing it.
113 * The zlib compression workset is initialized here to minimize
114 * the "per file" load. The jcr member is only set, if the init
117 jcr->compress_buf_size = jcr->buf_size + ((jcr->buf_size+999) / 1000) + 30;
118 jcr->compress_buf = get_memory(jcr->compress_buf_size);
121 z_stream *pZlibStream = (z_stream*)malloc(sizeof(z_stream));
123 pZlibStream->zalloc = Z_NULL;
124 pZlibStream->zfree = Z_NULL;
125 pZlibStream->opaque = Z_NULL;
126 pZlibStream->state = Z_NULL;
128 if (deflateInit(pZlibStream, Z_DEFAULT_COMPRESSION) == Z_OK) {
129 jcr->pZLIB_compress_workset = pZlibStream;
136 if (!crypto_session_start(jcr)) {
140 set_find_options((FF_PKT *)jcr->ff, jcr->incremental, jcr->mtime);
142 /* in accurate mode, we overwrite the find_one check function */
144 set_find_changed_function((FF_PKT *)jcr->ff, accurate_check_file);
147 start_heartbeat_monitor(jcr);
150 jcr->acl_data = (acl_data_t *)malloc(sizeof(acl_data_t));
151 memset((caddr_t)jcr->acl_data, 0, sizeof(acl_data_t));
152 jcr->acl_data->content = get_pool_memory(PM_MESSAGE);
156 jcr->xattr_data = (xattr_data_t *)malloc(sizeof(xattr_data_t));
157 memset((caddr_t)jcr->xattr_data, 0, sizeof(xattr_data_t));
158 jcr->xattr_data->content = get_pool_memory(PM_MESSAGE);
161 /* Subroutine save_file() is called for each file */
162 if (!find_files(jcr, (FF_PKT *)jcr->ff, save_file, plugin_save)) {
163 ok = false; /* error */
164 set_jcr_job_status(jcr, JS_ErrorTerminated);
167 if (have_acl && jcr->acl_data->nr_errors > 0) {
168 Jmsg(jcr, M_ERROR, 0, _("Encountered %ld acl errors while doing backup\n"),
169 jcr->acl_data->nr_errors);
171 if (have_xattr && jcr->xattr_data->nr_errors > 0) {
172 Jmsg(jcr, M_ERROR, 0, _("Encountered %ld xattr errors while doing backup\n"),
173 jcr->xattr_data->nr_errors);
176 accurate_finish(jcr); /* send deleted or base file list to SD */
178 stop_heartbeat_monitor(jcr);
180 sd->signal(BNET_EOD); /* end of sending data */
182 if (have_acl && jcr->acl_data) {
183 free_pool_memory(jcr->acl_data->content);
185 jcr->acl_data = NULL;
187 if (have_xattr && jcr->xattr_data) {
188 free_pool_memory(jcr->xattr_data->content);
189 free(jcr->xattr_data);
190 jcr->xattr_data = NULL;
196 if (jcr->compress_buf) {
197 free_pool_memory(jcr->compress_buf);
198 jcr->compress_buf = NULL;
200 if (jcr->pZLIB_compress_workset) {
201 /* Free the zlib stream */
203 deflateEnd((z_stream *)jcr->pZLIB_compress_workset);
205 free (jcr->pZLIB_compress_workset);
206 jcr->pZLIB_compress_workset = NULL;
208 crypto_session_end(jcr);
211 Dmsg1(100, "end blast_data ok=%d\n", ok);
215 static bool crypto_session_start(JCR *jcr)
217 crypto_cipher_t cipher = CRYPTO_CIPHER_AES_128_CBC;
220 * Create encryption session data and a cached, DER-encoded session data
221 * structure. We use a single session key for each backup, so we'll encode
222 * the session data only once.
224 if (jcr->crypto.pki_encrypt) {
227 /* Create per-job session encryption context */
228 jcr->crypto.pki_session = crypto_session_new(cipher, jcr->crypto.pki_recipients);
230 /* Get the session data size */
231 if (!crypto_session_encode(jcr->crypto.pki_session, (uint8_t *)0, &size)) {
232 Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n"));
236 /* Allocate buffer */
237 jcr->crypto.pki_session_encoded = get_memory(size);
239 /* Encode session data */
240 if (!crypto_session_encode(jcr->crypto.pki_session, (uint8_t *)jcr->crypto.pki_session_encoded, &size)) {
241 Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n"));
245 /* ... and store the encoded size */
246 jcr->crypto.pki_session_encoded_size = size;
248 /* Allocate the encryption/decryption buffer */
249 jcr->crypto.crypto_buf = get_memory(CRYPTO_CIPHER_MAX_BLOCK_SIZE);
254 static void crypto_session_end(JCR *jcr)
256 if (jcr->crypto.crypto_buf) {
257 free_pool_memory(jcr->crypto.crypto_buf);
258 jcr->crypto.crypto_buf = NULL;
260 if (jcr->crypto.pki_session) {
261 crypto_session_free(jcr->crypto.pki_session);
263 if (jcr->crypto.pki_session_encoded) {
264 free_pool_memory(jcr->crypto.pki_session_encoded);
265 jcr->crypto.pki_session_encoded = NULL;
269 static bool crypto_session_send(JCR *jcr, BSOCK *sd)
273 /* Send our header */
274 Dmsg2(100, "Send hdr fi=%ld stream=%d\n", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA);
275 sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA);
278 sd->msg = jcr->crypto.pki_session_encoded;
279 sd->msglen = jcr->crypto.pki_session_encoded_size;
280 jcr->JobBytes += sd->msglen;
282 Dmsg1(100, "Send data len=%d\n", sd->msglen);
285 sd->signal(BNET_EOD);
291 * Called here by find() for each file included.
292 * This is a callback. The original is find_files() above.
294 * Send the file and its data to the Storage daemon.
298 * -1 to ignore file/directory (not used here)
300 int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level)
302 bool do_read = false;
303 int stat, data_stream;
305 DIGEST *digest = NULL;
306 DIGEST *signing_digest = NULL;
307 int digest_stream = STREAM_NONE;
308 SIGNATURE *sig = NULL;
309 bool has_file_data = false;
310 // TODO landonf: Allow the user to specify the digest algorithm
312 crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA256;
314 crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA1;
316 BSOCK *sd = jcr->store_bsock;
318 if (job_canceled(jcr)) {
322 jcr->num_files_examined++; /* bump total file count */
324 switch (ff_pkt->type) {
325 case FT_LNKSAVED: /* Hard linked, file already saved */
326 Dmsg2(130, "FT_LNKSAVED hard link: %s => %s\n", ff_pkt->fname, ff_pkt->link);
329 Dmsg1(130, "FT_REGE saving: %s\n", ff_pkt->fname);
330 has_file_data = true;
333 Dmsg1(130, "FT_REG saving: %s\n", ff_pkt->fname);
334 has_file_data = true;
337 Dmsg2(130, "FT_LNK saving: %s -> %s\n", ff_pkt->fname, ff_pkt->link);
340 jcr->num_files_examined--; /* correct file count */
341 return 1; /* not used */
343 Jmsg(jcr, M_INFO, 1, _(" Recursion turned off. Will not descend from %s into %s\n"),
344 ff_pkt->top_fname, ff_pkt->fname);
345 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
348 /* Suppress message for /dev filesystems */
349 if (!is_in_fileset(ff_pkt)) {
350 Jmsg(jcr, M_INFO, 1, _(" %s is a different filesystem. Will not descend from %s into %s\n"),
351 ff_pkt->fname, ff_pkt->top_fname, ff_pkt->fname);
353 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
356 Jmsg(jcr, M_INFO, 1, _(" Disallowed filesystem. Will not descend from %s into %s\n"),
357 ff_pkt->top_fname, ff_pkt->fname);
358 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
361 Jmsg(jcr, M_INFO, 1, _(" Disallowed drive type. Will not descend into %s\n"),
366 Dmsg1(130, "FT_DIREND: %s\n", ff_pkt->link);
369 Dmsg1(130, "FT_SPEC saving: %s\n", ff_pkt->fname);
370 if (S_ISSOCK(ff_pkt->statp.st_mode)) {
371 Jmsg(jcr, M_SKIPPED, 1, _(" Socket file skipped: %s\n"), ff_pkt->fname);
376 Dmsg1(130, "FT_RAW saving: %s\n", ff_pkt->fname);
377 has_file_data = true;
380 Dmsg1(130, "FT_FIFO saving: %s\n", ff_pkt->fname);
384 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not access \"%s\": ERR=%s\n"), ff_pkt->fname,
385 be.bstrerror(ff_pkt->ff_errno));
391 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not follow link \"%s\": ERR=%s\n"),
392 ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
398 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not stat \"%s\": ERR=%s\n"), ff_pkt->fname,
399 be.bstrerror(ff_pkt->ff_errno));
405 Jmsg(jcr, M_SKIPPED, 1, _(" Unchanged file skipped: %s\n"), ff_pkt->fname);
408 Jmsg(jcr, M_NOTSAVED, 0, _(" Archive file not saved: %s\n"), ff_pkt->fname);
412 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not open directory \"%s\": ERR=%s\n"),
413 ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
418 Jmsg(jcr, M_NOTSAVED, 0, _(" Unknown file type %d; not saved: %s\n"),
419 ff_pkt->type, ff_pkt->fname);
424 Dmsg1(130, "bfiled: sending %s to stored\n", ff_pkt->fname);
426 /* Digests and encryption are only useful if there's file data */
429 * Setup for digest handling. If this fails, the digest will be set to NULL
430 * and not used. Note, the digest (file hash) can be any one of the four
433 * The signing digest is a single algorithm depending on
434 * whether or not we have SHA2.
435 * ****FIXME**** the signing algoritm should really be
436 * determined a different way!!!!!! What happens if
437 * sha2 was available during backup but not restore?
439 if (ff_pkt->flags & FO_MD5) {
440 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_MD5);
441 digest_stream = STREAM_MD5_DIGEST;
443 } else if (ff_pkt->flags & FO_SHA1) {
444 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA1);
445 digest_stream = STREAM_SHA1_DIGEST;
447 } else if (ff_pkt->flags & FO_SHA256) {
448 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA256);
449 digest_stream = STREAM_SHA256_DIGEST;
451 } else if (ff_pkt->flags & FO_SHA512) {
452 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA512);
453 digest_stream = STREAM_SHA512_DIGEST;
456 /* Did digest initialization fail? */
457 if (digest_stream != STREAM_NONE && digest == NULL) {
458 Jmsg(jcr, M_WARNING, 0, _("%s digest initialization failed\n"),
459 stream_to_ascii(digest_stream));
463 * Set up signature digest handling. If this fails, the signature digest
464 * will be set to NULL and not used.
466 /* TODO landonf: We should really only calculate the digest once, for
467 * both verification and signing.
469 if (jcr->crypto.pki_sign) {
470 signing_digest = crypto_digest_new(jcr, signing_algorithm);
472 /* Full-stop if a failure occurred initializing the signature digest */
473 if (signing_digest == NULL) {
474 Jmsg(jcr, M_NOTSAVED, 0, _("%s signature digest initialization failed\n"),
475 stream_to_ascii(signing_algorithm));
481 /* Enable encryption */
482 if (jcr->crypto.pki_encrypt) {
483 ff_pkt->flags |= FO_ENCRYPT;
487 /* Initialize the file descriptor we use for data and other streams. */
489 if (ff_pkt->flags & FO_PORTABLE) {
490 set_portable_backup(&ff_pkt->bfd); /* disable Win32 BackupRead() */
492 if (ff_pkt->cmd_plugin) {
493 if (!set_cmd_plugin(&ff_pkt->bfd, jcr)) {
496 send_plugin_name(jcr, sd, true); /* signal start of plugin data */
499 /* Send attributes -- must be done after binit() */
500 if (!encode_and_send_attributes(jcr, ff_pkt, data_stream)) {
504 /* Set up the encryption context and send the session data to the SD */
505 if (has_file_data && jcr->crypto.pki_encrypt) {
506 if (!crypto_session_send(jcr, sd)) {
512 * Open any file with data that we intend to save, then save it.
514 * Note, if is_win32_backup, we must open the Directory so that
515 * the BackupRead will save its permissions and ownership streams.
517 if (ff_pkt->type != FT_LNKSAVED && S_ISREG(ff_pkt->statp.st_mode)) {
519 do_read = !is_portable_backup(&ff_pkt->bfd) || ff_pkt->statp.st_size > 0;
521 do_read = ff_pkt->statp.st_size > 0;
523 } else if (ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO ||
524 ff_pkt->type == FT_REPARSE ||
525 (!is_portable_backup(&ff_pkt->bfd) && ff_pkt->type == FT_DIREND)) {
528 if (ff_pkt->cmd_plugin) {
532 Dmsg1(400, "do_read=%d\n", do_read);
536 if (ff_pkt->type == FT_FIFO) {
537 tid = start_thread_timer(jcr, pthread_self(), 60);
541 int noatime = ff_pkt->flags & FO_NOATIME ? O_NOATIME : 0;
542 ff_pkt->bfd.reparse_point = ff_pkt->type == FT_REPARSE;
543 if (bopen(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY | noatime, 0) < 0) {
544 ff_pkt->ff_errno = errno;
546 Jmsg(jcr, M_NOTSAVED, 0, _(" Cannot open \"%s\": ERR=%s.\n"), ff_pkt->fname,
550 stop_thread_timer(tid);
556 stop_thread_timer(tid);
560 stat = send_data(jcr, data_stream, ff_pkt, digest, signing_digest);
562 if (ff_pkt->flags & FO_CHKCHANGES) {
563 has_file_changed(jcr, ff_pkt);
566 bclose(&ff_pkt->bfd);
573 if (have_darwin_os) {
574 /* Regular files can have resource forks and Finder Info */
575 if (ff_pkt->type != FT_LNKSAVED && (S_ISREG(ff_pkt->statp.st_mode) &&
576 ff_pkt->flags & FO_HFSPLUS)) {
577 if (ff_pkt->hfsinfo.rsrclength > 0) {
580 if (!bopen_rsrc(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0) < 0) {
581 ff_pkt->ff_errno = errno;
583 Jmsg(jcr, M_NOTSAVED, -1, _(" Cannot open resource fork for \"%s\": ERR=%s.\n"),
584 ff_pkt->fname, be.bstrerror());
586 if (is_bopen(&ff_pkt->bfd)) {
587 bclose(&ff_pkt->bfd);
591 flags = ff_pkt->flags;
592 ff_pkt->flags &= ~(FO_GZIP|FO_SPARSE);
593 if (flags & FO_ENCRYPT) {
594 rsrc_stream = STREAM_ENCRYPTED_MACOS_FORK_DATA;
596 rsrc_stream = STREAM_MACOS_FORK_DATA;
598 stat = send_data(jcr, rsrc_stream, ff_pkt, digest, signing_digest);
599 ff_pkt->flags = flags;
600 bclose(&ff_pkt->bfd);
606 Dmsg1(300, "Saving Finder Info for \"%s\"\n", ff_pkt->fname);
607 sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_HFSPLUS_ATTRIBUTES);
608 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
609 pm_memcpy(sd->msg, ff_pkt->hfsinfo.fndrinfo, 32);
612 crypto_digest_update(digest, (uint8_t *)sd->msg, sd->msglen);
614 if (signing_digest) {
615 crypto_digest_update(signing_digest, (uint8_t *)sd->msg, sd->msglen);
618 sd->signal(BNET_EOD);
623 * Save ACLs when requested and available for anything not being a symlink and not being a plugin.
626 if (ff_pkt->flags & FO_ACL && ff_pkt->type != FT_LNK && !ff_pkt->cmd_plugin) {
627 switch (build_acl_streams(jcr, ff_pkt)) {
628 case bacl_exit_fatal:
630 case bacl_exit_error:
632 * Non-fatal errors, count them and when the number is under ACL_REPORT_ERR_MAX_PER_JOB
633 * print the error message set by the lower level routine in jcr->errmsg.
635 if (jcr->acl_data->nr_errors < ACL_REPORT_ERR_MAX_PER_JOB) {
636 Jmsg(jcr, M_ERROR, 0, "%s", jcr->errmsg);
638 jcr->acl_data->nr_errors++;
647 * Save Extended Attributes when requested and available for all files not being a plugin.
650 if (ff_pkt->flags & FO_XATTR && !ff_pkt->cmd_plugin) {
651 switch (build_xattr_streams(jcr, ff_pkt)) {
652 case bxattr_exit_fatal:
654 case bxattr_exit_error:
656 * Non-fatal errors, count them and when the number is under XATTR_REPORT_ERR_MAX_PER_JOB
657 * print the error message set by the lower level routine in jcr->errmsg.
659 if (jcr->xattr_data->nr_errors < XATTR_REPORT_ERR_MAX_PER_JOB) {
660 Jmsg(jcr, M_ERROR, 0, "%s", jcr->errmsg);
662 jcr->xattr_data->nr_errors++;
670 /* Terminate the signing digest and send it to the Storage daemon */
671 if (signing_digest) {
674 if ((sig = crypto_sign_new(jcr)) == NULL) {
675 Jmsg(jcr, M_FATAL, 0, _("Failed to allocate memory for crypto signature.\n"));
679 if (!crypto_sign_add_signer(sig, signing_digest, jcr->crypto.pki_keypair)) {
680 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
684 /* Get signature size */
685 if (!crypto_sign_encode(sig, NULL, &size)) {
686 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
690 /* Grow the bsock buffer to fit our message if necessary */
691 if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
692 sd->msg = realloc_pool_memory(sd->msg, size);
695 /* Send our header */
696 sd->fsend("%ld %ld 0", jcr->JobFiles, STREAM_SIGNED_DIGEST);
697 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
699 /* Encode signature data */
700 if (!crypto_sign_encode(sig, (uint8_t *)sd->msg, &size)) {
701 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
707 sd->signal(BNET_EOD); /* end of checksum */
710 /* Terminate any digest and send it to Storage daemon */
714 sd->fsend("%ld %d 0", jcr->JobFiles, digest_stream);
715 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
717 size = CRYPTO_DIGEST_MAX_SIZE;
719 /* Grow the bsock buffer to fit our message if necessary */
720 if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
721 sd->msg = realloc_pool_memory(sd->msg, size);
724 if (!crypto_digest_finalize(digest, (uint8_t *)sd->msg, &size)) {
725 Jmsg(jcr, M_FATAL, 0, _("An error occurred finalizing signing the stream.\n"));
731 sd->signal(BNET_EOD); /* end of checksum */
733 if (ff_pkt->cmd_plugin) {
734 send_plugin_name(jcr, sd, false); /* signal end of plugin data */
738 rtnstat = 1; /* good return */
742 crypto_digest_free(digest);
744 if (signing_digest) {
745 crypto_digest_free(signing_digest);
748 crypto_sign_free(sig);
754 * Send data read from an already open file descriptor.
756 * We return 1 on sucess and 0 on errors.
759 * We use ff_pkt->statp.st_size when FO_SPARSE to know when to stop
761 * Currently this is not a problem as the only other stream, resource forks,
762 * are not handled as sparse files.
764 static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest,
765 DIGEST *signing_digest)
767 BSOCK *sd = jcr->store_bsock;
768 uint64_t fileAddr = 0; /* file address */
770 int32_t rsize = jcr->buf_size; /* read buffer size */
772 CIPHER_CONTEXT *cipher_ctx = NULL; /* Quell bogus uninitialized warnings */
773 const uint8_t *cipher_input;
774 uint32_t cipher_input_len;
775 uint32_t cipher_block_size;
776 uint32_t encrypted_len;
777 #ifdef FD_NO_SEND_TEST
782 rbuf = sd->msg; /* read buffer */
783 wbuf = sd->msg; /* write buffer */
784 cipher_input = (uint8_t *)rbuf; /* encrypt uncompressed data */
786 Dmsg1(300, "Saving data, type=%d\n", ff_pkt->type);
789 uLong compress_len = 0;
790 uLong max_compress_len = 0;
791 const Bytef *cbuf = NULL;
794 if (ff_pkt->flags & FO_GZIP) {
795 if (ff_pkt->flags & FO_SPARSE) {
796 cbuf = (Bytef *)jcr->compress_buf + SPARSE_FADDR_SIZE;
797 max_compress_len = jcr->compress_buf_size - SPARSE_FADDR_SIZE;
799 cbuf = (Bytef *)jcr->compress_buf;
800 max_compress_len = jcr->compress_buf_size; /* set max length */
802 wbuf = jcr->compress_buf; /* compressed output here */
803 cipher_input = (uint8_t *)jcr->compress_buf; /* encrypt compressed data */
806 * Only change zlib parameters if there is no pending operation.
807 * This should never happen as deflatereset is called after each
811 if (((z_stream*)jcr->pZLIB_compress_workset)->total_in == 0) {
812 /* set gzip compression level - must be done per file */
813 if ((zstat=deflateParams((z_stream*)jcr->pZLIB_compress_workset,
814 ff_pkt->GZIP_level, Z_DEFAULT_STRATEGY)) != Z_OK) {
815 Jmsg(jcr, M_FATAL, 0, _("Compression deflateParams error: %d\n"), zstat);
816 set_jcr_job_status(jcr, JS_ErrorTerminated);
822 const uint32_t max_compress_len = 0;
825 if (ff_pkt->flags & FO_ENCRYPT) {
826 if (ff_pkt->flags & FO_SPARSE) {
827 Jmsg0(jcr, M_FATAL, 0, _("Encrypting sparse data not supported.\n"));
830 /* Allocate the cipher context */
831 if ((cipher_ctx = crypto_cipher_new(jcr->crypto.pki_session, true,
832 &cipher_block_size)) == NULL) {
833 /* Shouldn't happen! */
834 Jmsg0(jcr, M_FATAL, 0, _("Failed to initialize encryption context.\n"));
839 * Grow the crypto buffer, if necessary.
840 * crypto_cipher_update() will buffer up to (cipher_block_size - 1).
841 * We grow crypto_buf to the maximum number of blocks that
842 * could be returned for the given read buffer size.
843 * (Using the larger of either rsize or max_compress_len)
845 jcr->crypto.crypto_buf = check_pool_memory_size(jcr->crypto.crypto_buf,
846 (MAX(rsize + (int)sizeof(uint32_t), (int32_t)max_compress_len) +
847 cipher_block_size - 1) / cipher_block_size * cipher_block_size);
849 wbuf = jcr->crypto.crypto_buf; /* Encrypted, possibly compressed output here. */
853 * Send Data header to Storage daemon
854 * <file-index> <stream> <info>
856 if (!sd->fsend("%ld %d 0", jcr->JobFiles, stream)) {
857 if (!job_canceled(jcr)) {
858 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
863 Dmsg1(300, ">stored: datahdr %s\n", sd->msg);
866 * Make space at beginning of buffer for fileAddr because this
867 * same buffer will be used for writing if compression is off.
869 if (ff_pkt->flags & FO_SPARSE) {
870 rbuf += SPARSE_FADDR_SIZE;
871 rsize -= SPARSE_FADDR_SIZE;
872 #ifdef HAVE_FREEBSD_OS
874 * To read FreeBSD partitions, the read size must be
877 rsize = (rsize/512) * 512;
881 /* a RAW device read on win32 only works if the buffer is a multiple of 512 */
883 if (S_ISBLK(ff_pkt->statp.st_mode))
884 rsize = (rsize/512) * 512;
890 while ((sd->msglen=(uint32_t)bread(&ff_pkt->bfd, rbuf, rsize)) > 0) {
892 /* Check for sparse blocks */
893 if (ff_pkt->flags & FO_SPARSE) {
895 bool allZeros = false;
896 if ((sd->msglen == rsize &&
897 fileAddr+sd->msglen < (uint64_t)ff_pkt->statp.st_size) ||
898 ((ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO) &&
899 (uint64_t)ff_pkt->statp.st_size == 0)) {
900 allZeros = is_buf_zero(rbuf, rsize);
903 /* Put file address as first data in buffer */
904 ser_begin(wbuf, SPARSE_FADDR_SIZE);
905 ser_uint64(fileAddr); /* store fileAddr in begin of buffer */
907 fileAddr += sd->msglen; /* update file address */
908 /* Skip block of all zeros */
910 continue; /* skip block of zeros */
914 jcr->ReadBytes += sd->msglen; /* count bytes read */
916 /* Uncompressed cipher input length */
917 cipher_input_len = sd->msglen;
919 /* Update checksum if requested */
921 crypto_digest_update(digest, (uint8_t *)rbuf, sd->msglen);
924 /* Update signing digest if requested */
925 if (signing_digest) {
926 crypto_digest_update(signing_digest, (uint8_t *)rbuf, sd->msglen);
930 /* Do compression if turned on */
931 if (ff_pkt->flags & FO_GZIP && jcr->pZLIB_compress_workset) {
932 Dmsg3(400, "cbuf=0x%x rbuf=0x%x len=%u\n", cbuf, rbuf, sd->msglen);
934 ((z_stream*)jcr->pZLIB_compress_workset)->next_in = (Bytef *)rbuf;
935 ((z_stream*)jcr->pZLIB_compress_workset)->avail_in = sd->msglen;
936 ((z_stream*)jcr->pZLIB_compress_workset)->next_out = (Bytef *)cbuf;
937 ((z_stream*)jcr->pZLIB_compress_workset)->avail_out = max_compress_len;
939 if ((zstat=deflate((z_stream*)jcr->pZLIB_compress_workset, Z_FINISH)) != Z_STREAM_END) {
940 Jmsg(jcr, M_FATAL, 0, _("Compression deflate error: %d\n"), zstat);
941 set_jcr_job_status(jcr, JS_ErrorTerminated);
944 compress_len = ((z_stream*)jcr->pZLIB_compress_workset)->total_out;
945 /* reset zlib stream to be able to begin from scratch again */
946 if ((zstat=deflateReset((z_stream*)jcr->pZLIB_compress_workset)) != Z_OK) {
947 Jmsg(jcr, M_FATAL, 0, _("Compression deflateReset error: %d\n"), zstat);
948 set_jcr_job_status(jcr, JS_ErrorTerminated);
952 Dmsg2(400, "compressed len=%d uncompressed len=%d\n", compress_len,
955 sd->msglen = compress_len; /* set compressed length */
956 cipher_input_len = compress_len;
960 * Note, here we prepend the current record length to the beginning
961 * of the encrypted data. This is because both sparse and compression
962 * restore handling want records returned to them with exactly the
963 * same number of bytes that were processed in the backup handling.
964 * That is, both are block filters rather than a stream. When doing
965 * compression, the compression routines may buffer data, so that for
966 * any one record compressed, when it is decompressed the same size
967 * will not be obtained. Of course, the buffered data eventually comes
968 * out in subsequent crypto_cipher_update() calls or at least
969 * when crypto_cipher_finalize() is called. Unfortunately, this
970 * "feature" of encryption enormously complicates the restore code.
972 if (ff_pkt->flags & FO_ENCRYPT) {
973 uint32_t initial_len = 0;
976 if (ff_pkt->flags & FO_SPARSE) {
977 cipher_input_len += SPARSE_FADDR_SIZE;
980 /* Encrypt the length of the input block */
981 uint8_t packet_len[sizeof(uint32_t)];
983 ser_begin(packet_len, sizeof(uint32_t));
984 ser_uint32(cipher_input_len); /* store data len in begin of buffer */
985 Dmsg1(20, "Encrypt len=%d\n", cipher_input_len);
987 if (!crypto_cipher_update(cipher_ctx, packet_len, sizeof(packet_len),
988 (uint8_t *)jcr->crypto.crypto_buf, &initial_len)) {
989 /* Encryption failed. Shouldn't happen. */
990 Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
994 /* Encrypt the input block */
995 if (crypto_cipher_update(cipher_ctx, cipher_input, cipher_input_len,
996 (uint8_t *)&jcr->crypto.crypto_buf[initial_len], &encrypted_len)) {
997 if ((initial_len + encrypted_len) == 0) {
998 /* No full block of data available, read more data */
1001 Dmsg2(400, "encrypted len=%d unencrypted len=%d\n", encrypted_len,
1003 sd->msglen = initial_len + encrypted_len; /* set encrypted length */
1005 /* Encryption failed. Shouldn't happen. */
1006 Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
1011 /* Send the buffer to the Storage daemon */
1012 if (ff_pkt->flags & FO_SPARSE) {
1013 sd->msglen += SPARSE_FADDR_SIZE; /* include fileAddr in size */
1015 sd->msg = wbuf; /* set correct write buffer */
1017 if (!job_canceled(jcr)) {
1018 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1023 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
1025 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
1026 sd->msg = msgsave; /* restore read buffer */
1028 } /* end while read file data */
1030 if (sd->msglen < 0) { /* error */
1032 Jmsg(jcr, M_ERROR, 0, _("Read error on file %s. ERR=%s\n"),
1033 ff_pkt->fname, be.bstrerror(ff_pkt->bfd.berrno));
1034 if (jcr->JobErrors++ > 1000) { /* insanity check */
1035 Jmsg(jcr, M_FATAL, 0, _("Too many errors.\n"));
1037 } else if (ff_pkt->flags & FO_ENCRYPT) {
1039 * For encryption, we must call finalize to push out any
1042 if (!crypto_cipher_finalize(cipher_ctx, (uint8_t *)jcr->crypto.crypto_buf,
1044 /* Padding failed. Shouldn't happen. */
1045 Jmsg(jcr, M_FATAL, 0, _("Encryption padding error\n"));
1049 /* Note, on SSL pre-0.9.7, there is always some output */
1050 if (encrypted_len > 0) {
1051 sd->msglen = encrypted_len; /* set encrypted length */
1052 sd->msg = jcr->crypto.crypto_buf; /* set correct write buffer */
1054 if (!job_canceled(jcr)) {
1055 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1060 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
1061 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
1062 sd->msg = msgsave; /* restore bnet buffer */
1066 if (!sd->signal(BNET_EOD)) { /* indicate end of file data */
1067 if (!job_canceled(jcr)) {
1068 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1074 /* Free the cipher context */
1076 crypto_cipher_free(cipher_ctx);
1081 /* Free the cipher context */
1083 crypto_cipher_free(cipher_ctx);
1086 sd->msg = msgsave; /* restore bnet buffer */
1091 bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream)
1093 BSOCK *sd = jcr->store_bsock;
1094 char attribs[MAXSTRING];
1095 char attribsEx[MAXSTRING];
1098 #ifdef FD_NO_SEND_TEST
1102 Dmsg1(300, "encode_and_send_attrs fname=%s\n", ff_pkt->fname);
1103 /* Find what data stream we will use, then encode the attributes */
1104 if ((data_stream = select_data_stream(ff_pkt)) == STREAM_NONE) {
1105 /* This should not happen */
1106 Jmsg0(jcr, M_FATAL, 0, _("Invalid file flags, no supported data stream type.\n"));
1109 encode_stat(attribs, &ff_pkt->statp, ff_pkt->LinkFI, data_stream);
1111 /* Now possibly extend the attributes */
1112 attr_stream = encode_attribsEx(jcr, attribsEx, ff_pkt);
1114 Dmsg3(300, "File %s\nattribs=%s\nattribsEx=%s\n", ff_pkt->fname, attribs, attribsEx);
1117 jcr->JobFiles++; /* increment number of files sent */
1118 ff_pkt->FileIndex = jcr->JobFiles; /* return FileIndex */
1119 pm_strcpy(jcr->last_fname, ff_pkt->fname);
1123 * Send Attributes header to Storage daemon
1124 * <file-index> <stream> <info>
1126 if (!sd->fsend("%ld %d 0", jcr->JobFiles, attr_stream)) {
1127 if (!job_canceled(jcr)) {
1128 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1133 Dmsg1(300, ">stored: attrhdr %s\n", sd->msg);
1136 * Send file attributes to Storage daemon
1139 * Filename (full path)
1140 * Encoded attributes
1141 * Link name (if type==FT_LNK or FT_LNKSAVED)
1142 * Encoded extended-attributes (for Win32)
1144 * For a directory, link is the same as fname, but with trailing
1145 * slash. For a linked file, link is the link.
1147 if (ff_pkt->type != FT_DELETED) { /* already stripped */
1150 if (ff_pkt->type == FT_LNK || ff_pkt->type == FT_LNKSAVED) {
1151 Dmsg2(300, "Link %s to %s\n", ff_pkt->fname, ff_pkt->link);
1152 stat = sd->fsend("%ld %d %s%c%s%c%s%c%s%c", jcr->JobFiles,
1153 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, ff_pkt->link, 0,
1155 } else if (ff_pkt->type == FT_DIREND || ff_pkt->type == FT_REPARSE) {
1156 /* Here link is the canonical filename (i.e. with trailing slash) */
1157 stat = sd->fsend("%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
1158 ff_pkt->type, ff_pkt->link, 0, attribs, 0, 0, attribsEx, 0);
1160 stat = sd->fsend("%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
1161 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, 0, attribsEx, 0);
1163 if (ff_pkt->type != FT_DELETED) {
1164 unstrip_path(ff_pkt);
1167 Dmsg2(300, ">stored: attr len=%d: %s\n", sd->msglen, sd->msg);
1169 if (!job_canceled(jcr)) {
1170 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1175 sd->signal(BNET_EOD); /* indicate end of attributes data */
1180 * Do in place strip of path
1182 static bool do_strip(int count, char *in)
1188 /* Copy to first path separator -- Win32 might have c: ... */
1189 while (*in && !IsPathSeparator(*in)) {
1193 numsep++; /* one separator seen */
1194 for (stripped=0; stripped<count && *in; stripped++) {
1195 while (*in && !IsPathSeparator(*in)) {
1196 in++; /* skip chars */
1199 numsep++; /* count separators seen */
1200 in++; /* skip separator */
1204 while (*in) { /* copy to end */
1205 if (IsPathSeparator(*in)) {
1211 Dmsg4(500, "stripped=%d count=%d numsep=%d sep>count=%d\n",
1212 stripped, count, numsep, numsep>count);
1213 return stripped==count && numsep>count;
1217 * If requested strip leading components of the path so that we can
1218 * save file as if it came from a subdirectory. This is most useful
1219 * for dealing with snapshots, by removing the snapshot directory, or
1220 * in handling vendor migrations where files have been restored with
1221 * a vendor product into a subdirectory.
1223 void strip_path(FF_PKT *ff_pkt)
1225 if (!(ff_pkt->flags & FO_STRIPPATH) || ff_pkt->strip_path <= 0) {
1226 Dmsg1(200, "No strip for %s\n", ff_pkt->fname);
1229 if (!ff_pkt->fname_save) {
1230 ff_pkt->fname_save = get_pool_memory(PM_FNAME);
1231 ff_pkt->link_save = get_pool_memory(PM_FNAME);
1233 pm_strcpy(ff_pkt->fname_save, ff_pkt->fname);
1234 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1235 pm_strcpy(ff_pkt->link_save, ff_pkt->link);
1236 Dmsg2(500, "strcpy link_save=%d link=%d\n", strlen(ff_pkt->link_save),
1237 strlen(ff_pkt->link));
1238 sm_check(__FILE__, __LINE__, true);
1242 * Strip path. If it doesn't succeed put it back. If
1243 * it does, and there is a different link string,
1244 * attempt to strip the link. If it fails, back them
1246 * Do not strip symlinks.
1247 * I.e. if either stripping fails don't strip anything.
1249 if (!do_strip(ff_pkt->strip_path, ff_pkt->fname)) {
1250 unstrip_path(ff_pkt);
1253 /* Strip links but not symlinks */
1254 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1255 if (!do_strip(ff_pkt->strip_path, ff_pkt->link)) {
1256 unstrip_path(ff_pkt);
1261 Dmsg3(100, "fname=%s stripped=%s link=%s\n", ff_pkt->fname_save, ff_pkt->fname,
1265 void unstrip_path(FF_PKT *ff_pkt)
1267 if (!(ff_pkt->flags & FO_STRIPPATH) || ff_pkt->strip_path <= 0) {
1270 strcpy(ff_pkt->fname, ff_pkt->fname_save);
1271 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1272 Dmsg2(500, "strcpy link=%s link_save=%s\n", ff_pkt->link,
1274 strcpy(ff_pkt->link, ff_pkt->link_save);
1275 Dmsg2(500, "strcpy link=%d link_save=%d\n", strlen(ff_pkt->link),
1276 strlen(ff_pkt->link_save));
1277 sm_check(__FILE__, __LINE__, true);