2 Bacula® - The Network Backup Solution
4 Copyright (C) 2000-2010 Free Software Foundation Europe e.V.
6 The main author of Bacula is Kern Sibbald, with contributions from
7 many others, a complete list can be found in the file AUTHORS.
8 This program is Free Software; you can redistribute it and/or
9 modify it under the terms of version two of the GNU General Public
10 License as published by the Free Software Foundation and included
13 This program is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
23 Bacula® is a registered trademark of Kern Sibbald.
24 The licensor of Bacula is the Free Software Foundation Europe
25 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
26 Switzerland, email:ftf@fsfeurope.org.
29 * Bacula File Daemon backup.c send file attributes and data
30 * to the Storage daemon.
32 * Kern Sibbald, March MM
40 const bool have_darwin_os = true;
42 const bool have_darwin_os = false;
46 const bool have_acl = true;
48 const bool have_acl = false;
51 #if defined(HAVE_XATTR)
52 const bool have_xattr = true;
54 const bool have_xattr = false;
57 /* Forward referenced functions */
58 int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level);
59 static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, DIGEST *signature_digest);
60 bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream);
61 static bool crypto_session_start(JCR *jcr);
62 static void crypto_session_end(JCR *jcr);
63 static bool crypto_session_send(JCR *jcr, BSOCK *sd);
66 * Find all the requested files and send them
67 * to the Storage daemon.
69 * Note, we normally carry on a one-way
70 * conversation from this point on with the SD, simply blasting
71 * data to him. To properly know what is going on, we
72 * also run a "heartbeat" monitor which reads the socket and
73 * reacts accordingly (at the moment it has nothing to do
74 * except echo the heartbeat to the Director).
77 bool blast_data_to_storage_daemon(JCR *jcr, char *addr)
81 // TODO landonf: Allow user to specify encryption algorithm
83 sd = jcr->store_bsock;
85 set_jcr_job_status(jcr, JS_Running);
87 Dmsg1(300, "bfiled: opened data connection %d to stored\n", sd->m_fd);
90 CLIENT *client = (CLIENT *)GetNextRes(R_CLIENT, NULL);
94 buf_size = client->max_network_buffer_size;
96 buf_size = 0; /* use default */
98 if (!sd->set_buffer_size(buf_size, BNET_SETBUF_WRITE)) {
99 set_jcr_job_status(jcr, JS_ErrorTerminated);
100 Jmsg(jcr, M_FATAL, 0, _("Cannot set buffer size FD->SD.\n"));
104 jcr->buf_size = sd->msglen;
106 * Adjust for compression so that output buffer is
107 * 12 bytes + 0.1% larger than input buffer plus 18 bytes.
108 * This gives a bit extra plus room for the sparse addr if any.
109 * Note, we adjust the read size to be smaller so that the
110 * same output buffer can be used without growing it.
112 * The zlib compression workset is initialized here to minimize
113 * the "per file" load. The jcr member is only set, if the init
116 jcr->compress_buf_size = jcr->buf_size + ((jcr->buf_size+999) / 1000) + 30;
117 jcr->compress_buf = get_memory(jcr->compress_buf_size);
120 z_stream *pZlibStream = (z_stream*)malloc(sizeof(z_stream));
122 pZlibStream->zalloc = Z_NULL;
123 pZlibStream->zfree = Z_NULL;
124 pZlibStream->opaque = Z_NULL;
125 pZlibStream->state = Z_NULL;
127 if (deflateInit(pZlibStream, Z_DEFAULT_COMPRESSION) == Z_OK) {
128 jcr->pZLIB_compress_workset = pZlibStream;
135 if (!crypto_session_start(jcr)) {
139 set_find_options((FF_PKT *)jcr->ff, jcr->incremental, jcr->mtime);
141 /** in accurate mode, we overwrite the find_one check function */
143 set_find_changed_function((FF_PKT *)jcr->ff, accurate_check_file);
146 start_heartbeat_monitor(jcr);
149 jcr->acl_data = (acl_data_t *)malloc(sizeof(acl_data_t));
150 memset((caddr_t)jcr->acl_data, 0, sizeof(acl_data_t));
151 jcr->acl_data->content = get_pool_memory(PM_MESSAGE);
155 jcr->xattr_data = (xattr_data_t *)malloc(sizeof(xattr_data_t));
156 memset((caddr_t)jcr->xattr_data, 0, sizeof(xattr_data_t));
157 jcr->xattr_data->content = get_pool_memory(PM_MESSAGE);
160 /** Subroutine save_file() is called for each file */
161 if (!find_files(jcr, (FF_PKT *)jcr->ff, save_file, plugin_save)) {
162 ok = false; /* error */
163 set_jcr_job_status(jcr, JS_ErrorTerminated);
166 if (have_acl && jcr->acl_data->nr_errors > 0) {
167 Jmsg(jcr, M_ERROR, 0, _("Encountered %ld acl errors while doing backup\n"),
168 jcr->acl_data->nr_errors);
170 if (have_xattr && jcr->xattr_data->nr_errors > 0) {
171 Jmsg(jcr, M_ERROR, 0, _("Encountered %ld xattr errors while doing backup\n"),
172 jcr->xattr_data->nr_errors);
175 accurate_finish(jcr); /* send deleted or base file list to SD */
177 stop_heartbeat_monitor(jcr);
179 sd->signal(BNET_EOD); /* end of sending data */
181 if (have_acl && jcr->acl_data) {
182 free_pool_memory(jcr->acl_data->content);
184 jcr->acl_data = NULL;
186 if (have_xattr && jcr->xattr_data) {
187 free_pool_memory(jcr->xattr_data->content);
188 free(jcr->xattr_data);
189 jcr->xattr_data = NULL;
195 if (jcr->compress_buf) {
196 free_pool_memory(jcr->compress_buf);
197 jcr->compress_buf = NULL;
199 if (jcr->pZLIB_compress_workset) {
200 /* Free the zlib stream */
202 deflateEnd((z_stream *)jcr->pZLIB_compress_workset);
204 free (jcr->pZLIB_compress_workset);
205 jcr->pZLIB_compress_workset = NULL;
207 crypto_session_end(jcr);
210 Dmsg1(100, "end blast_data ok=%d\n", ok);
214 static bool crypto_session_start(JCR *jcr)
216 crypto_cipher_t cipher = CRYPTO_CIPHER_AES_128_CBC;
219 * Create encryption session data and a cached, DER-encoded session data
220 * structure. We use a single session key for each backup, so we'll encode
221 * the session data only once.
223 if (jcr->crypto.pki_encrypt) {
226 /** Create per-job session encryption context */
227 jcr->crypto.pki_session = crypto_session_new(cipher, jcr->crypto.pki_recipients);
229 /** Get the session data size */
230 if (!crypto_session_encode(jcr->crypto.pki_session, (uint8_t *)0, &size)) {
231 Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n"));
235 /** Allocate buffer */
236 jcr->crypto.pki_session_encoded = get_memory(size);
238 /** Encode session data */
239 if (!crypto_session_encode(jcr->crypto.pki_session, (uint8_t *)jcr->crypto.pki_session_encoded, &size)) {
240 Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n"));
244 /** ... and store the encoded size */
245 jcr->crypto.pki_session_encoded_size = size;
247 /** Allocate the encryption/decryption buffer */
248 jcr->crypto.crypto_buf = get_memory(CRYPTO_CIPHER_MAX_BLOCK_SIZE);
253 static void crypto_session_end(JCR *jcr)
255 if (jcr->crypto.crypto_buf) {
256 free_pool_memory(jcr->crypto.crypto_buf);
257 jcr->crypto.crypto_buf = NULL;
259 if (jcr->crypto.pki_session) {
260 crypto_session_free(jcr->crypto.pki_session);
262 if (jcr->crypto.pki_session_encoded) {
263 free_pool_memory(jcr->crypto.pki_session_encoded);
264 jcr->crypto.pki_session_encoded = NULL;
268 static bool crypto_session_send(JCR *jcr, BSOCK *sd)
272 /** Send our header */
273 Dmsg2(100, "Send hdr fi=%ld stream=%d\n", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA);
274 sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA);
277 sd->msg = jcr->crypto.pki_session_encoded;
278 sd->msglen = jcr->crypto.pki_session_encoded_size;
279 jcr->JobBytes += sd->msglen;
281 Dmsg1(100, "Send data len=%d\n", sd->msglen);
284 sd->signal(BNET_EOD);
290 * Called here by find() for each file included.
291 * This is a callback. The original is find_files() above.
293 * Send the file and its data to the Storage daemon.
297 * -1 to ignore file/directory (not used here)
299 int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level)
301 bool do_read = false;
302 int stat, data_stream;
304 DIGEST *digest = NULL;
305 DIGEST *signing_digest = NULL;
306 int digest_stream = STREAM_NONE;
307 SIGNATURE *sig = NULL;
308 bool has_file_data = false;
309 // TODO landonf: Allow the user to specify the digest algorithm
311 crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA256;
313 crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA1;
315 BSOCK *sd = jcr->store_bsock;
317 if (jcr->is_job_canceled()) {
321 jcr->num_files_examined++; /* bump total file count */
323 switch (ff_pkt->type) {
324 case FT_LNKSAVED: /* Hard linked, file already saved */
325 Dmsg2(130, "FT_LNKSAVED hard link: %s => %s\n", ff_pkt->fname, ff_pkt->link);
328 Dmsg1(130, "FT_REGE saving: %s\n", ff_pkt->fname);
329 has_file_data = true;
332 Dmsg1(130, "FT_REG saving: %s\n", ff_pkt->fname);
333 has_file_data = true;
336 Dmsg2(130, "FT_LNK saving: %s -> %s\n", ff_pkt->fname, ff_pkt->link);
338 case FT_RESTORE_FIRST:
339 Dmsg1(00, "FT_RESTORE_FIRST saving: %s\n", ff_pkt->fname);
342 jcr->num_files_examined--; /* correct file count */
343 return 1; /* not used */
345 Jmsg(jcr, M_INFO, 1, _(" Recursion turned off. Will not descend from %s into %s\n"),
346 ff_pkt->top_fname, ff_pkt->fname);
347 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
350 /* Suppress message for /dev filesystems */
351 if (!is_in_fileset(ff_pkt)) {
352 Jmsg(jcr, M_INFO, 1, _(" %s is a different filesystem. Will not descend from %s into %s\n"),
353 ff_pkt->fname, ff_pkt->top_fname, ff_pkt->fname);
355 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
358 Jmsg(jcr, M_INFO, 1, _(" Disallowed filesystem. Will not descend from %s into %s\n"),
359 ff_pkt->top_fname, ff_pkt->fname);
360 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
363 Jmsg(jcr, M_INFO, 1, _(" Disallowed drive type. Will not descend into %s\n"),
368 Dmsg1(130, "FT_DIREND: %s\n", ff_pkt->link);
371 Dmsg1(130, "FT_SPEC saving: %s\n", ff_pkt->fname);
372 if (S_ISSOCK(ff_pkt->statp.st_mode)) {
373 Jmsg(jcr, M_SKIPPED, 1, _(" Socket file skipped: %s\n"), ff_pkt->fname);
378 Dmsg1(130, "FT_RAW saving: %s\n", ff_pkt->fname);
379 has_file_data = true;
382 Dmsg1(130, "FT_FIFO saving: %s\n", ff_pkt->fname);
386 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not access \"%s\": ERR=%s\n"), ff_pkt->fname,
387 be.bstrerror(ff_pkt->ff_errno));
393 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not follow link \"%s\": ERR=%s\n"),
394 ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
400 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not stat \"%s\": ERR=%s\n"), ff_pkt->fname,
401 be.bstrerror(ff_pkt->ff_errno));
407 Jmsg(jcr, M_SKIPPED, 1, _(" Unchanged file skipped: %s\n"), ff_pkt->fname);
410 Jmsg(jcr, M_NOTSAVED, 0, _(" Archive file not saved: %s\n"), ff_pkt->fname);
414 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not open directory \"%s\": ERR=%s\n"),
415 ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
420 Jmsg(jcr, M_NOTSAVED, 0, _(" Unknown file type %d; not saved: %s\n"),
421 ff_pkt->type, ff_pkt->fname);
426 Dmsg1(130, "bfiled: sending %s to stored\n", ff_pkt->fname);
428 /** Digests and encryption are only useful if there's file data */
431 * Setup for digest handling. If this fails, the digest will be set to NULL
432 * and not used. Note, the digest (file hash) can be any one of the four
435 * The signing digest is a single algorithm depending on
436 * whether or not we have SHA2.
437 * ****FIXME**** the signing algoritm should really be
438 * determined a different way!!!!!! What happens if
439 * sha2 was available during backup but not restore?
441 if (ff_pkt->flags & FO_MD5) {
442 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_MD5);
443 digest_stream = STREAM_MD5_DIGEST;
445 } else if (ff_pkt->flags & FO_SHA1) {
446 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA1);
447 digest_stream = STREAM_SHA1_DIGEST;
449 } else if (ff_pkt->flags & FO_SHA256) {
450 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA256);
451 digest_stream = STREAM_SHA256_DIGEST;
453 } else if (ff_pkt->flags & FO_SHA512) {
454 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA512);
455 digest_stream = STREAM_SHA512_DIGEST;
458 /** Did digest initialization fail? */
459 if (digest_stream != STREAM_NONE && digest == NULL) {
460 Jmsg(jcr, M_WARNING, 0, _("%s digest initialization failed\n"),
461 stream_to_ascii(digest_stream));
465 * Set up signature digest handling. If this fails, the signature digest
466 * will be set to NULL and not used.
468 /* TODO landonf: We should really only calculate the digest once, for
469 * both verification and signing.
471 if (jcr->crypto.pki_sign) {
472 signing_digest = crypto_digest_new(jcr, signing_algorithm);
474 /** Full-stop if a failure occurred initializing the signature digest */
475 if (signing_digest == NULL) {
476 Jmsg(jcr, M_NOTSAVED, 0, _("%s signature digest initialization failed\n"),
477 stream_to_ascii(signing_algorithm));
483 /** Enable encryption */
484 if (jcr->crypto.pki_encrypt) {
485 ff_pkt->flags |= FO_ENCRYPT;
489 /** Initialize the file descriptor we use for data and other streams. */
491 if (ff_pkt->flags & FO_PORTABLE) {
492 set_portable_backup(&ff_pkt->bfd); /* disable Win32 BackupRead() */
494 if (ff_pkt->cmd_plugin) {
495 /* Tell bfile that it needs to call plugin */
496 if (!set_cmd_plugin(&ff_pkt->bfd, jcr)) {
499 send_plugin_name(jcr, sd, true); /* signal start of plugin data */
502 /** Send attributes -- must be done after binit() */
503 if (!encode_and_send_attributes(jcr, ff_pkt, data_stream)) {
507 /** Set up the encryption context and send the session data to the SD */
508 if (has_file_data && jcr->crypto.pki_encrypt) {
509 if (!crypto_session_send(jcr, sd)) {
515 * Open any file with data that we intend to save, then save it.
517 * Note, if is_win32_backup, we must open the Directory so that
518 * the BackupRead will save its permissions and ownership streams.
520 if (ff_pkt->type != FT_LNKSAVED && S_ISREG(ff_pkt->statp.st_mode)) {
522 do_read = !is_portable_backup(&ff_pkt->bfd) || ff_pkt->statp.st_size > 0;
524 do_read = ff_pkt->statp.st_size > 0;
526 } else if (ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO ||
527 ff_pkt->type == FT_REPARSE ||
528 (!is_portable_backup(&ff_pkt->bfd) && ff_pkt->type == FT_DIREND)) {
531 if (ff_pkt->cmd_plugin) {
535 Dmsg1(400, "do_read=%d\n", do_read);
539 if (ff_pkt->type == FT_FIFO) {
540 tid = start_thread_timer(jcr, pthread_self(), 60);
544 int noatime = ff_pkt->flags & FO_NOATIME ? O_NOATIME : 0;
545 ff_pkt->bfd.reparse_point = ff_pkt->type == FT_REPARSE;
546 if (bopen(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY | noatime, 0) < 0) {
547 ff_pkt->ff_errno = errno;
549 Jmsg(jcr, M_NOTSAVED, 0, _(" Cannot open \"%s\": ERR=%s.\n"), ff_pkt->fname,
553 stop_thread_timer(tid);
559 stop_thread_timer(tid);
563 stat = send_data(jcr, data_stream, ff_pkt, digest, signing_digest);
565 if (ff_pkt->flags & FO_CHKCHANGES) {
566 has_file_changed(jcr, ff_pkt);
569 bclose(&ff_pkt->bfd);
576 if (have_darwin_os) {
577 /** Regular files can have resource forks and Finder Info */
578 if (ff_pkt->type != FT_LNKSAVED && (S_ISREG(ff_pkt->statp.st_mode) &&
579 ff_pkt->flags & FO_HFSPLUS)) {
580 if (ff_pkt->hfsinfo.rsrclength > 0) {
583 if (!bopen_rsrc(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0) < 0) {
584 ff_pkt->ff_errno = errno;
586 Jmsg(jcr, M_NOTSAVED, -1, _(" Cannot open resource fork for \"%s\": ERR=%s.\n"),
587 ff_pkt->fname, be.bstrerror());
589 if (is_bopen(&ff_pkt->bfd)) {
590 bclose(&ff_pkt->bfd);
594 flags = ff_pkt->flags;
595 ff_pkt->flags &= ~(FO_GZIP|FO_SPARSE);
596 if (flags & FO_ENCRYPT) {
597 rsrc_stream = STREAM_ENCRYPTED_MACOS_FORK_DATA;
599 rsrc_stream = STREAM_MACOS_FORK_DATA;
601 stat = send_data(jcr, rsrc_stream, ff_pkt, digest, signing_digest);
602 ff_pkt->flags = flags;
603 bclose(&ff_pkt->bfd);
609 Dmsg1(300, "Saving Finder Info for \"%s\"\n", ff_pkt->fname);
610 sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_HFSPLUS_ATTRIBUTES);
611 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
612 pm_memcpy(sd->msg, ff_pkt->hfsinfo.fndrinfo, 32);
615 crypto_digest_update(digest, (uint8_t *)sd->msg, sd->msglen);
617 if (signing_digest) {
618 crypto_digest_update(signing_digest, (uint8_t *)sd->msg, sd->msglen);
621 sd->signal(BNET_EOD);
626 * Save ACLs when requested and available for anything not being a symlink and not being a plugin.
629 if (ff_pkt->flags & FO_ACL && ff_pkt->type != FT_LNK && !ff_pkt->cmd_plugin) {
630 switch (build_acl_streams(jcr, ff_pkt)) {
631 case bacl_exit_fatal:
633 case bacl_exit_error:
635 * Non-fatal errors, count them and when the number is under ACL_REPORT_ERR_MAX_PER_JOB
636 * print the error message set by the lower level routine in jcr->errmsg.
638 if (jcr->acl_data->nr_errors < ACL_REPORT_ERR_MAX_PER_JOB) {
639 Jmsg(jcr, M_ERROR, 0, "%s", jcr->errmsg);
641 jcr->acl_data->nr_errors++;
650 * Save Extended Attributes when requested and available for all files not being a plugin.
653 if (ff_pkt->flags & FO_XATTR && !ff_pkt->cmd_plugin) {
654 switch (build_xattr_streams(jcr, ff_pkt)) {
655 case bxattr_exit_fatal:
657 case bxattr_exit_error:
659 * Non-fatal errors, count them and when the number is under XATTR_REPORT_ERR_MAX_PER_JOB
660 * print the error message set by the lower level routine in jcr->errmsg.
662 if (jcr->xattr_data->nr_errors < XATTR_REPORT_ERR_MAX_PER_JOB) {
663 Jmsg(jcr, M_ERROR, 0, "%s", jcr->errmsg);
665 jcr->xattr_data->nr_errors++;
673 /** Terminate the signing digest and send it to the Storage daemon */
674 if (signing_digest) {
677 if ((sig = crypto_sign_new(jcr)) == NULL) {
678 Jmsg(jcr, M_FATAL, 0, _("Failed to allocate memory for crypto signature.\n"));
682 if (!crypto_sign_add_signer(sig, signing_digest, jcr->crypto.pki_keypair)) {
683 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
687 /** Get signature size */
688 if (!crypto_sign_encode(sig, NULL, &size)) {
689 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
693 /** Grow the bsock buffer to fit our message if necessary */
694 if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
695 sd->msg = realloc_pool_memory(sd->msg, size);
698 /** Send our header */
699 sd->fsend("%ld %ld 0", jcr->JobFiles, STREAM_SIGNED_DIGEST);
700 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
702 /** Encode signature data */
703 if (!crypto_sign_encode(sig, (uint8_t *)sd->msg, &size)) {
704 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
710 sd->signal(BNET_EOD); /* end of checksum */
713 /** Terminate any digest and send it to Storage daemon */
717 sd->fsend("%ld %d 0", jcr->JobFiles, digest_stream);
718 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
720 size = CRYPTO_DIGEST_MAX_SIZE;
722 /** Grow the bsock buffer to fit our message if necessary */
723 if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
724 sd->msg = realloc_pool_memory(sd->msg, size);
727 if (!crypto_digest_finalize(digest, (uint8_t *)sd->msg, &size)) {
728 Jmsg(jcr, M_FATAL, 0, _("An error occurred finalizing signing the stream.\n"));
734 sd->signal(BNET_EOD); /* end of checksum */
736 if (ff_pkt->cmd_plugin) {
737 send_plugin_name(jcr, sd, false); /* signal end of plugin data */
741 rtnstat = 1; /* good return */
745 crypto_digest_free(digest);
747 if (signing_digest) {
748 crypto_digest_free(signing_digest);
751 crypto_sign_free(sig);
757 * Send data read from an already open file descriptor.
759 * We return 1 on sucess and 0 on errors.
762 * We use ff_pkt->statp.st_size when FO_SPARSE to know when to stop
764 * Currently this is not a problem as the only other stream, resource forks,
765 * are not handled as sparse files.
767 static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest,
768 DIGEST *signing_digest)
770 BSOCK *sd = jcr->store_bsock;
771 uint64_t fileAddr = 0; /* file address */
773 int32_t rsize = jcr->buf_size; /* read buffer size */
775 CIPHER_CONTEXT *cipher_ctx = NULL; /* Quell bogus uninitialized warnings */
776 const uint8_t *cipher_input;
777 uint32_t cipher_input_len;
778 uint32_t cipher_block_size;
779 uint32_t encrypted_len;
780 #ifdef FD_NO_SEND_TEST
785 rbuf = sd->msg; /* read buffer */
786 wbuf = sd->msg; /* write buffer */
787 cipher_input = (uint8_t *)rbuf; /* encrypt uncompressed data */
789 Dmsg1(300, "Saving data, type=%d\n", ff_pkt->type);
792 uLong compress_len = 0;
793 uLong max_compress_len = 0;
794 const Bytef *cbuf = NULL;
797 if (ff_pkt->flags & FO_GZIP) {
798 if (ff_pkt->flags & FO_SPARSE) {
799 cbuf = (Bytef *)jcr->compress_buf + SPARSE_FADDR_SIZE;
800 max_compress_len = jcr->compress_buf_size - SPARSE_FADDR_SIZE;
802 cbuf = (Bytef *)jcr->compress_buf;
803 max_compress_len = jcr->compress_buf_size; /* set max length */
805 wbuf = jcr->compress_buf; /* compressed output here */
806 cipher_input = (uint8_t *)jcr->compress_buf; /* encrypt compressed data */
809 * Only change zlib parameters if there is no pending operation.
810 * This should never happen as deflatereset is called after each
814 if (((z_stream*)jcr->pZLIB_compress_workset)->total_in == 0) {
815 /** set gzip compression level - must be done per file */
816 if ((zstat=deflateParams((z_stream*)jcr->pZLIB_compress_workset,
817 ff_pkt->GZIP_level, Z_DEFAULT_STRATEGY)) != Z_OK) {
818 Jmsg(jcr, M_FATAL, 0, _("Compression deflateParams error: %d\n"), zstat);
819 set_jcr_job_status(jcr, JS_ErrorTerminated);
825 const uint32_t max_compress_len = 0;
828 if (ff_pkt->flags & FO_ENCRYPT) {
829 if (ff_pkt->flags & FO_SPARSE) {
830 Jmsg0(jcr, M_FATAL, 0, _("Encrypting sparse data not supported.\n"));
833 /** Allocate the cipher context */
834 if ((cipher_ctx = crypto_cipher_new(jcr->crypto.pki_session, true,
835 &cipher_block_size)) == NULL) {
836 /* Shouldn't happen! */
837 Jmsg0(jcr, M_FATAL, 0, _("Failed to initialize encryption context.\n"));
842 * Grow the crypto buffer, if necessary.
843 * crypto_cipher_update() will buffer up to (cipher_block_size - 1).
844 * We grow crypto_buf to the maximum number of blocks that
845 * could be returned for the given read buffer size.
846 * (Using the larger of either rsize or max_compress_len)
848 jcr->crypto.crypto_buf = check_pool_memory_size(jcr->crypto.crypto_buf,
849 (MAX(rsize + (int)sizeof(uint32_t), (int32_t)max_compress_len) +
850 cipher_block_size - 1) / cipher_block_size * cipher_block_size);
852 wbuf = jcr->crypto.crypto_buf; /* Encrypted, possibly compressed output here. */
856 * Send Data header to Storage daemon
857 * <file-index> <stream> <info>
859 if (!sd->fsend("%ld %d 0", jcr->JobFiles, stream)) {
860 if (!jcr->is_job_canceled()) {
861 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
866 Dmsg1(300, ">stored: datahdr %s\n", sd->msg);
869 * Make space at beginning of buffer for fileAddr because this
870 * same buffer will be used for writing if compression is off.
872 if (ff_pkt->flags & FO_SPARSE) {
873 rbuf += SPARSE_FADDR_SIZE;
874 rsize -= SPARSE_FADDR_SIZE;
875 #ifdef HAVE_FREEBSD_OS
877 * To read FreeBSD partitions, the read size must be
880 rsize = (rsize/512) * 512;
884 /** a RAW device read on win32 only works if the buffer is a multiple of 512 */
886 if (S_ISBLK(ff_pkt->statp.st_mode))
887 rsize = (rsize/512) * 512;
893 while ((sd->msglen=(uint32_t)bread(&ff_pkt->bfd, rbuf, rsize)) > 0) {
895 /** Check for sparse blocks */
896 if (ff_pkt->flags & FO_SPARSE) {
898 bool allZeros = false;
899 if ((sd->msglen == rsize &&
900 fileAddr+sd->msglen < (uint64_t)ff_pkt->statp.st_size) ||
901 ((ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO) &&
902 (uint64_t)ff_pkt->statp.st_size == 0)) {
903 allZeros = is_buf_zero(rbuf, rsize);
906 /** Put file address as first data in buffer */
907 ser_begin(wbuf, SPARSE_FADDR_SIZE);
908 ser_uint64(fileAddr); /* store fileAddr in begin of buffer */
910 fileAddr += sd->msglen; /* update file address */
911 /** Skip block of all zeros */
913 continue; /* skip block of zeros */
917 jcr->ReadBytes += sd->msglen; /* count bytes read */
919 /** Uncompressed cipher input length */
920 cipher_input_len = sd->msglen;
922 /** Update checksum if requested */
924 crypto_digest_update(digest, (uint8_t *)rbuf, sd->msglen);
927 /** Update signing digest if requested */
928 if (signing_digest) {
929 crypto_digest_update(signing_digest, (uint8_t *)rbuf, sd->msglen);
933 /** Do compression if turned on */
934 if (ff_pkt->flags & FO_GZIP && jcr->pZLIB_compress_workset) {
935 Dmsg3(400, "cbuf=0x%x rbuf=0x%x len=%u\n", cbuf, rbuf, sd->msglen);
937 ((z_stream*)jcr->pZLIB_compress_workset)->next_in = (Bytef *)rbuf;
938 ((z_stream*)jcr->pZLIB_compress_workset)->avail_in = sd->msglen;
939 ((z_stream*)jcr->pZLIB_compress_workset)->next_out = (Bytef *)cbuf;
940 ((z_stream*)jcr->pZLIB_compress_workset)->avail_out = max_compress_len;
942 if ((zstat=deflate((z_stream*)jcr->pZLIB_compress_workset, Z_FINISH)) != Z_STREAM_END) {
943 Jmsg(jcr, M_FATAL, 0, _("Compression deflate error: %d\n"), zstat);
944 set_jcr_job_status(jcr, JS_ErrorTerminated);
947 compress_len = ((z_stream*)jcr->pZLIB_compress_workset)->total_out;
948 /** reset zlib stream to be able to begin from scratch again */
949 if ((zstat=deflateReset((z_stream*)jcr->pZLIB_compress_workset)) != Z_OK) {
950 Jmsg(jcr, M_FATAL, 0, _("Compression deflateReset error: %d\n"), zstat);
951 set_jcr_job_status(jcr, JS_ErrorTerminated);
955 Dmsg2(400, "compressed len=%d uncompressed len=%d\n", compress_len,
958 sd->msglen = compress_len; /* set compressed length */
959 cipher_input_len = compress_len;
963 * Note, here we prepend the current record length to the beginning
964 * of the encrypted data. This is because both sparse and compression
965 * restore handling want records returned to them with exactly the
966 * same number of bytes that were processed in the backup handling.
967 * That is, both are block filters rather than a stream. When doing
968 * compression, the compression routines may buffer data, so that for
969 * any one record compressed, when it is decompressed the same size
970 * will not be obtained. Of course, the buffered data eventually comes
971 * out in subsequent crypto_cipher_update() calls or at least
972 * when crypto_cipher_finalize() is called. Unfortunately, this
973 * "feature" of encryption enormously complicates the restore code.
975 if (ff_pkt->flags & FO_ENCRYPT) {
976 uint32_t initial_len = 0;
979 if (ff_pkt->flags & FO_SPARSE) {
980 cipher_input_len += SPARSE_FADDR_SIZE;
983 /** Encrypt the length of the input block */
984 uint8_t packet_len[sizeof(uint32_t)];
986 ser_begin(packet_len, sizeof(uint32_t));
987 ser_uint32(cipher_input_len); /* store data len in begin of buffer */
988 Dmsg1(20, "Encrypt len=%d\n", cipher_input_len);
990 if (!crypto_cipher_update(cipher_ctx, packet_len, sizeof(packet_len),
991 (uint8_t *)jcr->crypto.crypto_buf, &initial_len)) {
992 /** Encryption failed. Shouldn't happen. */
993 Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
997 /** Encrypt the input block */
998 if (crypto_cipher_update(cipher_ctx, cipher_input, cipher_input_len,
999 (uint8_t *)&jcr->crypto.crypto_buf[initial_len], &encrypted_len)) {
1000 if ((initial_len + encrypted_len) == 0) {
1001 /** No full block of data available, read more data */
1004 Dmsg2(400, "encrypted len=%d unencrypted len=%d\n", encrypted_len,
1006 sd->msglen = initial_len + encrypted_len; /* set encrypted length */
1008 /** Encryption failed. Shouldn't happen. */
1009 Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
1014 /* Send the buffer to the Storage daemon */
1015 if (ff_pkt->flags & FO_SPARSE) {
1016 sd->msglen += SPARSE_FADDR_SIZE; /* include fileAddr in size */
1018 sd->msg = wbuf; /* set correct write buffer */
1020 if (!jcr->is_job_canceled()) {
1021 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1026 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
1028 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
1029 sd->msg = msgsave; /* restore read buffer */
1031 } /* end while read file data */
1033 if (sd->msglen < 0) { /* error */
1035 Jmsg(jcr, M_ERROR, 0, _("Read error on file %s. ERR=%s\n"),
1036 ff_pkt->fname, be.bstrerror(ff_pkt->bfd.berrno));
1037 if (jcr->JobErrors++ > 1000) { /* insanity check */
1038 Jmsg(jcr, M_FATAL, 0, _("Too many errors. JobErrors=%d.\n"), jcr->JobErrors);
1040 } else if (ff_pkt->flags & FO_ENCRYPT) {
1042 * For encryption, we must call finalize to push out any
1045 if (!crypto_cipher_finalize(cipher_ctx, (uint8_t *)jcr->crypto.crypto_buf,
1047 /* Padding failed. Shouldn't happen. */
1048 Jmsg(jcr, M_FATAL, 0, _("Encryption padding error\n"));
1052 /** Note, on SSL pre-0.9.7, there is always some output */
1053 if (encrypted_len > 0) {
1054 sd->msglen = encrypted_len; /* set encrypted length */
1055 sd->msg = jcr->crypto.crypto_buf; /* set correct write buffer */
1057 if (!jcr->is_job_canceled()) {
1058 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1063 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
1064 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
1065 sd->msg = msgsave; /* restore bnet buffer */
1069 if (!sd->signal(BNET_EOD)) { /* indicate end of file data */
1070 if (!jcr->is_job_canceled()) {
1071 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1077 /** Free the cipher context */
1079 crypto_cipher_free(cipher_ctx);
1084 /** Free the cipher context */
1086 crypto_cipher_free(cipher_ctx);
1089 sd->msg = msgsave; /* restore bnet buffer */
1094 bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream)
1096 BSOCK *sd = jcr->store_bsock;
1097 char attribs[MAXSTRING];
1098 char attribsEx[MAXSTRING];
1101 #ifdef FD_NO_SEND_TEST
1105 Dmsg1(300, "encode_and_send_attrs fname=%s\n", ff_pkt->fname);
1106 /** Find what data stream we will use, then encode the attributes */
1107 if ((data_stream = select_data_stream(ff_pkt)) == STREAM_NONE) {
1108 /* This should not happen */
1109 Jmsg0(jcr, M_FATAL, 0, _("Invalid file flags, no supported data stream type.\n"));
1112 encode_stat(attribs, &ff_pkt->statp, ff_pkt->LinkFI, data_stream);
1114 /** Now possibly extend the attributes */
1115 attr_stream = encode_attribsEx(jcr, attribsEx, ff_pkt);
1117 Dmsg3(300, "File %s\nattribs=%s\nattribsEx=%s\n", ff_pkt->fname, attribs, attribsEx);
1120 jcr->JobFiles++; /* increment number of files sent */
1121 ff_pkt->FileIndex = jcr->JobFiles; /* return FileIndex */
1122 pm_strcpy(jcr->last_fname, ff_pkt->fname);
1126 * Send Attributes header to Storage daemon
1127 * <file-index> <stream> <info>
1129 if (!sd->fsend("%ld %d 0", jcr->JobFiles, attr_stream)) {
1130 if (!jcr->is_job_canceled()) {
1131 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1136 Dmsg1(300, ">stored: attrhdr %s\n", sd->msg);
1139 * Send file attributes to Storage daemon
1142 * Filename (full path)
1143 * Encoded attributes
1144 * Link name (if type==FT_LNK or FT_LNKSAVED)
1145 * Encoded extended-attributes (for Win32)
1147 * For a directory, link is the same as fname, but with trailing
1148 * slash. For a linked file, link is the link.
1150 if (ff_pkt->type != FT_DELETED) { /* already stripped */
1153 switch (ff_pkt->type) {
1156 Dmsg2(300, "Link %s to %s\n", ff_pkt->fname, ff_pkt->link);
1157 stat = sd->fsend("%ld %d %s%c%s%c%s%c%s%c", jcr->JobFiles,
1158 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, ff_pkt->link, 0,
1163 /* Here link is the canonical filename (i.e. with trailing slash) */
1164 stat = sd->fsend("%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
1165 ff_pkt->type, ff_pkt->link, 0, attribs, 0, 0, attribsEx, 0);
1168 stat = sd->fsend("%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
1169 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, 0, attribsEx, 0);
1172 if (ff_pkt->type != FT_DELETED) {
1173 unstrip_path(ff_pkt);
1176 Dmsg2(300, ">stored: attr len=%d: %s\n", sd->msglen, sd->msg);
1177 if (!stat && !jcr->is_job_canceled()) {
1178 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1181 sd->signal(BNET_EOD); /* indicate end of attributes data */
1186 * Do in place strip of path
1188 static bool do_strip(int count, char *in)
1194 /** Copy to first path separator -- Win32 might have c: ... */
1195 while (*in && !IsPathSeparator(*in)) {
1199 numsep++; /* one separator seen */
1200 for (stripped=0; stripped<count && *in; stripped++) {
1201 while (*in && !IsPathSeparator(*in)) {
1202 in++; /* skip chars */
1205 numsep++; /* count separators seen */
1206 in++; /* skip separator */
1210 while (*in) { /* copy to end */
1211 if (IsPathSeparator(*in)) {
1217 Dmsg4(500, "stripped=%d count=%d numsep=%d sep>count=%d\n",
1218 stripped, count, numsep, numsep>count);
1219 return stripped==count && numsep>count;
1223 * If requested strip leading components of the path so that we can
1224 * save file as if it came from a subdirectory. This is most useful
1225 * for dealing with snapshots, by removing the snapshot directory, or
1226 * in handling vendor migrations where files have been restored with
1227 * a vendor product into a subdirectory.
1229 void strip_path(FF_PKT *ff_pkt)
1231 if (!(ff_pkt->flags & FO_STRIPPATH) || ff_pkt->strip_path <= 0) {
1232 Dmsg1(200, "No strip for %s\n", ff_pkt->fname);
1235 if (!ff_pkt->fname_save) {
1236 ff_pkt->fname_save = get_pool_memory(PM_FNAME);
1237 ff_pkt->link_save = get_pool_memory(PM_FNAME);
1239 pm_strcpy(ff_pkt->fname_save, ff_pkt->fname);
1240 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1241 pm_strcpy(ff_pkt->link_save, ff_pkt->link);
1242 Dmsg2(500, "strcpy link_save=%d link=%d\n", strlen(ff_pkt->link_save),
1243 strlen(ff_pkt->link));
1244 sm_check(__FILE__, __LINE__, true);
1248 * Strip path. If it doesn't succeed put it back. If
1249 * it does, and there is a different link string,
1250 * attempt to strip the link. If it fails, back them
1252 * Do not strip symlinks.
1253 * I.e. if either stripping fails don't strip anything.
1255 if (!do_strip(ff_pkt->strip_path, ff_pkt->fname)) {
1256 unstrip_path(ff_pkt);
1259 /** Strip links but not symlinks */
1260 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1261 if (!do_strip(ff_pkt->strip_path, ff_pkt->link)) {
1262 unstrip_path(ff_pkt);
1267 Dmsg3(100, "fname=%s stripped=%s link=%s\n", ff_pkt->fname_save, ff_pkt->fname,
1271 void unstrip_path(FF_PKT *ff_pkt)
1273 if (!(ff_pkt->flags & FO_STRIPPATH) || ff_pkt->strip_path <= 0) {
1276 strcpy(ff_pkt->fname, ff_pkt->fname_save);
1277 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1278 Dmsg2(500, "strcpy link=%s link_save=%s\n", ff_pkt->link,
1280 strcpy(ff_pkt->link, ff_pkt->link_save);
1281 Dmsg2(500, "strcpy link=%d link_save=%d\n", strlen(ff_pkt->link),
1282 strlen(ff_pkt->link_save));
1283 sm_check(__FILE__, __LINE__, true);