2 Bacula® - The Network Backup Solution
4 Copyright (C) 2000-2011 Free Software Foundation Europe e.V.
6 The main author of Bacula is Kern Sibbald, with contributions from
7 many others, a complete list can be found in the file AUTHORS.
8 This program is Free Software; you can redistribute it and/or
9 modify it under the terms of version three of the GNU Affero General Public
10 License as published by the Free Software Foundation and included
13 This program is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 General Public License for more details.
18 You should have received a copy of the GNU Affero General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
23 Bacula® is a registered trademark of Kern Sibbald.
24 The licensor of Bacula is the Free Software Foundation Europe
25 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
26 Switzerland, email:ftf@fsfeurope.org.
29 * Bacula File Daemon backup.c send file attributes and data
30 * to the Storage daemon.
32 * Kern Sibbald, March MM
40 const bool have_darwin_os = true;
42 const bool have_darwin_os = false;
46 const bool have_acl = true;
48 const bool have_acl = false;
51 #if defined(HAVE_XATTR)
52 const bool have_xattr = true;
54 const bool have_xattr = false;
57 /* Forward referenced functions */
58 int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level);
59 static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, DIGEST *signature_digest);
60 bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream);
61 static bool crypto_session_start(JCR *jcr);
62 static void crypto_session_end(JCR *jcr);
63 static bool crypto_session_send(JCR *jcr, BSOCK *sd);
64 static void close_vss_backup_session(JCR *jcr);
67 * Find all the requested files and send them
68 * to the Storage daemon.
70 * Note, we normally carry on a one-way
71 * conversation from this point on with the SD, simply blasting
72 * data to him. To properly know what is going on, we
73 * also run a "heartbeat" monitor which reads the socket and
74 * reacts accordingly (at the moment it has nothing to do
75 * except echo the heartbeat to the Director).
78 bool blast_data_to_storage_daemon(JCR *jcr, char *addr)
82 // TODO landonf: Allow user to specify encryption algorithm
84 sd = jcr->store_bsock;
86 set_jcr_job_status(jcr, JS_Running);
88 Dmsg1(300, "bfiled: opened data connection %d to stored\n", sd->m_fd);
91 CLIENT *client = (CLIENT *)GetNextRes(R_CLIENT, NULL);
95 buf_size = client->max_network_buffer_size;
97 buf_size = 0; /* use default */
99 if (!sd->set_buffer_size(buf_size, BNET_SETBUF_WRITE)) {
100 set_jcr_job_status(jcr, JS_ErrorTerminated);
101 Jmsg(jcr, M_FATAL, 0, _("Cannot set buffer size FD->SD.\n"));
105 jcr->buf_size = sd->msglen;
107 * Adjust for compression so that output buffer is
108 * 12 bytes + 0.1% larger than input buffer plus 18 bytes.
109 * This gives a bit extra plus room for the sparse addr if any.
110 * Note, we adjust the read size to be smaller so that the
111 * same output buffer can be used without growing it.
113 * The zlib compression workset is initialized here to minimize
114 * the "per file" load. The jcr member is only set, if the init
117 jcr->compress_buf_size = jcr->buf_size + ((jcr->buf_size+999) / 1000) + 30;
118 jcr->compress_buf = get_memory(jcr->compress_buf_size);
121 z_stream *pZlibStream = (z_stream*)malloc(sizeof(z_stream));
123 pZlibStream->zalloc = Z_NULL;
124 pZlibStream->zfree = Z_NULL;
125 pZlibStream->opaque = Z_NULL;
126 pZlibStream->state = Z_NULL;
128 if (deflateInit(pZlibStream, Z_DEFAULT_COMPRESSION) == Z_OK) {
129 jcr->pZLIB_compress_workset = pZlibStream;
136 if (!crypto_session_start(jcr)) {
140 set_find_options((FF_PKT *)jcr->ff, jcr->incremental, jcr->mtime);
142 /** in accurate mode, we overload the find_one check function */
144 set_find_changed_function((FF_PKT *)jcr->ff, accurate_check_file);
147 start_heartbeat_monitor(jcr);
150 jcr->acl_data = (acl_data_t *)malloc(sizeof(acl_data_t));
151 memset((caddr_t)jcr->acl_data, 0, sizeof(acl_data_t));
152 jcr->acl_data->content = get_pool_memory(PM_MESSAGE);
156 jcr->xattr_data = (xattr_data_t *)malloc(sizeof(xattr_data_t));
157 memset((caddr_t)jcr->xattr_data, 0, sizeof(xattr_data_t));
158 jcr->xattr_data->content = get_pool_memory(PM_MESSAGE);
161 /** Subroutine save_file() is called for each file */
162 if (!find_files(jcr, (FF_PKT *)jcr->ff, save_file, plugin_save)) {
163 ok = false; /* error */
164 set_jcr_job_status(jcr, JS_ErrorTerminated);
167 if (have_acl && jcr->acl_data->nr_errors > 0) {
168 Jmsg(jcr, M_ERROR, 0, _("Encountered %ld acl errors while doing backup\n"),
169 jcr->acl_data->nr_errors);
171 if (have_xattr && jcr->xattr_data->nr_errors > 0) {
172 Jmsg(jcr, M_ERROR, 0, _("Encountered %ld xattr errors while doing backup\n"),
173 jcr->xattr_data->nr_errors);
176 close_vss_backup_session(jcr);
178 accurate_finish(jcr); /* send deleted or base file list to SD */
180 stop_heartbeat_monitor(jcr);
182 sd->signal(BNET_EOD); /* end of sending data */
184 if (have_acl && jcr->acl_data) {
185 free_pool_memory(jcr->acl_data->content);
187 jcr->acl_data = NULL;
189 if (have_xattr && jcr->xattr_data) {
190 free_pool_memory(jcr->xattr_data->content);
191 free(jcr->xattr_data);
192 jcr->xattr_data = NULL;
198 if (jcr->compress_buf) {
199 free_pool_memory(jcr->compress_buf);
200 jcr->compress_buf = NULL;
202 if (jcr->pZLIB_compress_workset) {
203 /* Free the zlib stream */
205 deflateEnd((z_stream *)jcr->pZLIB_compress_workset);
207 free (jcr->pZLIB_compress_workset);
208 jcr->pZLIB_compress_workset = NULL;
210 crypto_session_end(jcr);
213 Dmsg1(100, "end blast_data ok=%d\n", ok);
217 static bool crypto_session_start(JCR *jcr)
219 crypto_cipher_t cipher = CRYPTO_CIPHER_AES_128_CBC;
222 * Create encryption session data and a cached, DER-encoded session data
223 * structure. We use a single session key for each backup, so we'll encode
224 * the session data only once.
226 if (jcr->crypto.pki_encrypt) {
229 /** Create per-job session encryption context */
230 jcr->crypto.pki_session = crypto_session_new(cipher, jcr->crypto.pki_recipients);
232 /** Get the session data size */
233 if (!crypto_session_encode(jcr->crypto.pki_session, (uint8_t *)0, &size)) {
234 Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n"));
238 /** Allocate buffer */
239 jcr->crypto.pki_session_encoded = get_memory(size);
241 /** Encode session data */
242 if (!crypto_session_encode(jcr->crypto.pki_session, (uint8_t *)jcr->crypto.pki_session_encoded, &size)) {
243 Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n"));
247 /** ... and store the encoded size */
248 jcr->crypto.pki_session_encoded_size = size;
250 /** Allocate the encryption/decryption buffer */
251 jcr->crypto.crypto_buf = get_memory(CRYPTO_CIPHER_MAX_BLOCK_SIZE);
256 static void crypto_session_end(JCR *jcr)
258 if (jcr->crypto.crypto_buf) {
259 free_pool_memory(jcr->crypto.crypto_buf);
260 jcr->crypto.crypto_buf = NULL;
262 if (jcr->crypto.pki_session) {
263 crypto_session_free(jcr->crypto.pki_session);
265 if (jcr->crypto.pki_session_encoded) {
266 free_pool_memory(jcr->crypto.pki_session_encoded);
267 jcr->crypto.pki_session_encoded = NULL;
271 static bool crypto_session_send(JCR *jcr, BSOCK *sd)
275 /** Send our header */
276 Dmsg2(100, "Send hdr fi=%ld stream=%d\n", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA);
277 sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA);
280 sd->msg = jcr->crypto.pki_session_encoded;
281 sd->msglen = jcr->crypto.pki_session_encoded_size;
282 jcr->JobBytes += sd->msglen;
284 Dmsg1(100, "Send data len=%d\n", sd->msglen);
287 sd->signal(BNET_EOD);
293 * Called here by find() for each file included.
294 * This is a callback. The original is find_files() above.
296 * Send the file and its data to the Storage daemon.
300 * -1 to ignore file/directory (not used here)
302 int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level)
304 bool do_read = false;
305 bool plugin_started = false;
306 int stat, data_stream;
308 DIGEST *digest = NULL;
309 DIGEST *signing_digest = NULL;
310 int digest_stream = STREAM_NONE;
311 SIGNATURE *sig = NULL;
312 bool has_file_data = false;
313 // TODO landonf: Allow the user to specify the digest algorithm
315 crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA256;
317 crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA1;
319 BSOCK *sd = jcr->store_bsock;
321 if (jcr->is_canceled() || jcr->is_incomplete()) {
325 jcr->num_files_examined++; /* bump total file count */
327 switch (ff_pkt->type) {
328 case FT_LNKSAVED: /* Hard linked, file already saved */
329 Dmsg2(130, "FT_LNKSAVED hard link: %s => %s\n", ff_pkt->fname, ff_pkt->link);
332 Dmsg1(130, "FT_REGE saving: %s\n", ff_pkt->fname);
333 has_file_data = true;
336 Dmsg1(130, "FT_REG saving: %s\n", ff_pkt->fname);
337 has_file_data = true;
340 Dmsg2(130, "FT_LNK saving: %s -> %s\n", ff_pkt->fname, ff_pkt->link);
342 case FT_RESTORE_FIRST:
343 Dmsg1(100, "FT_RESTORE_FIRST saving: %s\n", ff_pkt->fname);
346 jcr->num_files_examined--; /* correct file count */
347 return 1; /* not used */
349 Jmsg(jcr, M_INFO, 1, _(" Recursion turned off. Will not descend from %s into %s\n"),
350 ff_pkt->top_fname, ff_pkt->fname);
351 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
354 /* Suppress message for /dev filesystems */
355 if (!is_in_fileset(ff_pkt)) {
356 Jmsg(jcr, M_INFO, 1, _(" %s is a different filesystem. Will not descend from %s into it.\n"),
357 ff_pkt->fname, ff_pkt->top_fname);
359 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
362 Jmsg(jcr, M_INFO, 1, _(" Disallowed filesystem. Will not descend from %s into %s\n"),
363 ff_pkt->top_fname, ff_pkt->fname);
364 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
367 Jmsg(jcr, M_INFO, 1, _(" Disallowed drive type. Will not descend into %s\n"),
373 Dmsg1(130, "FT_DIREND: %s\n", ff_pkt->link);
376 Dmsg1(130, "FT_SPEC saving: %s\n", ff_pkt->fname);
377 if (S_ISSOCK(ff_pkt->statp.st_mode)) {
378 Jmsg(jcr, M_SKIPPED, 1, _(" Socket file skipped: %s\n"), ff_pkt->fname);
383 Dmsg1(130, "FT_RAW saving: %s\n", ff_pkt->fname);
384 has_file_data = true;
387 Dmsg1(130, "FT_FIFO saving: %s\n", ff_pkt->fname);
391 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not access \"%s\": ERR=%s\n"), ff_pkt->fname,
392 be.bstrerror(ff_pkt->ff_errno));
398 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not follow link \"%s\": ERR=%s\n"),
399 ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
405 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not stat \"%s\": ERR=%s\n"), ff_pkt->fname,
406 be.bstrerror(ff_pkt->ff_errno));
412 Jmsg(jcr, M_SKIPPED, 1, _(" Unchanged file skipped: %s\n"), ff_pkt->fname);
415 Jmsg(jcr, M_NOTSAVED, 0, _(" Archive file not saved: %s\n"), ff_pkt->fname);
419 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not open directory \"%s\": ERR=%s\n"),
420 ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
425 Jmsg(jcr, M_NOTSAVED, 0, _(" Unknown file type %d; not saved: %s\n"),
426 ff_pkt->type, ff_pkt->fname);
431 Dmsg1(130, "bfiled: sending %s to stored\n", ff_pkt->fname);
433 /** Digests and encryption are only useful if there's file data */
436 * Setup for digest handling. If this fails, the digest will be set to NULL
437 * and not used. Note, the digest (file hash) can be any one of the four
440 * The signing digest is a single algorithm depending on
441 * whether or not we have SHA2.
442 * ****FIXME**** the signing algoritm should really be
443 * determined a different way!!!!!! What happens if
444 * sha2 was available during backup but not restore?
446 if (ff_pkt->flags & FO_MD5) {
447 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_MD5);
448 digest_stream = STREAM_MD5_DIGEST;
450 } else if (ff_pkt->flags & FO_SHA1) {
451 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA1);
452 digest_stream = STREAM_SHA1_DIGEST;
454 } else if (ff_pkt->flags & FO_SHA256) {
455 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA256);
456 digest_stream = STREAM_SHA256_DIGEST;
458 } else if (ff_pkt->flags & FO_SHA512) {
459 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA512);
460 digest_stream = STREAM_SHA512_DIGEST;
463 /** Did digest initialization fail? */
464 if (digest_stream != STREAM_NONE && digest == NULL) {
465 Jmsg(jcr, M_WARNING, 0, _("%s digest initialization failed\n"),
466 stream_to_ascii(digest_stream));
470 * Set up signature digest handling. If this fails, the signature digest
471 * will be set to NULL and not used.
473 /* TODO landonf: We should really only calculate the digest once, for
474 * both verification and signing.
476 if (jcr->crypto.pki_sign) {
477 signing_digest = crypto_digest_new(jcr, signing_algorithm);
479 /** Full-stop if a failure occurred initializing the signature digest */
480 if (signing_digest == NULL) {
481 Jmsg(jcr, M_NOTSAVED, 0, _("%s signature digest initialization failed\n"),
482 stream_to_ascii(signing_algorithm));
488 /** Enable encryption */
489 if (jcr->crypto.pki_encrypt) {
490 ff_pkt->flags |= FO_ENCRYPT;
494 /** Initialize the file descriptor we use for data and other streams. */
496 if (ff_pkt->flags & FO_PORTABLE) {
497 set_portable_backup(&ff_pkt->bfd); /* disable Win32 BackupRead() */
499 if (ff_pkt->cmd_plugin) {
500 /* Tell bfile that it needs to call plugin */
501 if (!set_cmd_plugin(&ff_pkt->bfd, jcr)) {
504 send_plugin_name(jcr, sd, true); /* signal start of plugin data */
505 plugin_started = true;
508 /** Send attributes -- must be done after binit() */
509 if (!encode_and_send_attributes(jcr, ff_pkt, data_stream)) {
512 /** Meta data only for restore object */
513 if (ff_pkt->type == FT_RESTORE_FIRST) {
517 /** Set up the encryption context and send the session data to the SD */
518 if (has_file_data && jcr->crypto.pki_encrypt) {
519 if (!crypto_session_send(jcr, sd)) {
525 * Open any file with data that we intend to save, then save it.
527 * Note, if is_win32_backup, we must open the Directory so that
528 * the BackupRead will save its permissions and ownership streams.
530 if (ff_pkt->type != FT_LNKSAVED && S_ISREG(ff_pkt->statp.st_mode)) {
532 do_read = !is_portable_backup(&ff_pkt->bfd) || ff_pkt->statp.st_size > 0;
534 do_read = ff_pkt->statp.st_size > 0;
536 } else if (ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO ||
537 ff_pkt->type == FT_REPARSE || ff_pkt->type == FT_JUNCTION ||
538 (!is_portable_backup(&ff_pkt->bfd) && ff_pkt->type == FT_DIREND)) {
542 if (ff_pkt->cmd_plugin) {
546 Dmsg2(150, "type=%d do_read=%d\n", ff_pkt->type, do_read);
550 if (ff_pkt->type == FT_FIFO) {
551 tid = start_thread_timer(jcr, pthread_self(), 60);
555 int noatime = ff_pkt->flags & FO_NOATIME ? O_NOATIME : 0;
556 ff_pkt->bfd.reparse_point = (ff_pkt->type == FT_REPARSE ||
557 ff_pkt->type == FT_JUNCTION);
558 if (bopen(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY | noatime, 0) < 0) {
559 ff_pkt->ff_errno = errno;
561 Jmsg(jcr, M_NOTSAVED, 0, _(" Cannot open \"%s\": ERR=%s.\n"), ff_pkt->fname,
565 stop_thread_timer(tid);
571 stop_thread_timer(tid);
575 stat = send_data(jcr, data_stream, ff_pkt, digest, signing_digest);
577 if (ff_pkt->flags & FO_CHKCHANGES) {
578 has_file_changed(jcr, ff_pkt);
581 bclose(&ff_pkt->bfd);
588 if (have_darwin_os) {
589 /** Regular files can have resource forks and Finder Info */
590 if (ff_pkt->type != FT_LNKSAVED && (S_ISREG(ff_pkt->statp.st_mode) &&
591 ff_pkt->flags & FO_HFSPLUS)) {
592 if (ff_pkt->hfsinfo.rsrclength > 0) {
595 if (!bopen_rsrc(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0) < 0) {
596 ff_pkt->ff_errno = errno;
598 Jmsg(jcr, M_NOTSAVED, -1, _(" Cannot open resource fork for \"%s\": ERR=%s.\n"),
599 ff_pkt->fname, be.bstrerror());
601 if (is_bopen(&ff_pkt->bfd)) {
602 bclose(&ff_pkt->bfd);
606 flags = ff_pkt->flags;
607 ff_pkt->flags &= ~(FO_GZIP|FO_SPARSE|FO_OFFSETS);
608 if (flags & FO_ENCRYPT) {
609 rsrc_stream = STREAM_ENCRYPTED_MACOS_FORK_DATA;
611 rsrc_stream = STREAM_MACOS_FORK_DATA;
613 stat = send_data(jcr, rsrc_stream, ff_pkt, digest, signing_digest);
614 ff_pkt->flags = flags;
615 bclose(&ff_pkt->bfd);
621 Dmsg1(300, "Saving Finder Info for \"%s\"\n", ff_pkt->fname);
622 sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_HFSPLUS_ATTRIBUTES);
623 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
624 pm_memcpy(sd->msg, ff_pkt->hfsinfo.fndrinfo, 32);
627 crypto_digest_update(digest, (uint8_t *)sd->msg, sd->msglen);
629 if (signing_digest) {
630 crypto_digest_update(signing_digest, (uint8_t *)sd->msg, sd->msglen);
633 sd->signal(BNET_EOD);
638 * Save ACLs when requested and available for anything not being a symlink and not being a plugin.
641 if (ff_pkt->flags & FO_ACL && ff_pkt->type != FT_LNK && !ff_pkt->cmd_plugin) {
642 switch (build_acl_streams(jcr, ff_pkt)) {
643 case bacl_exit_fatal:
645 case bacl_exit_error:
647 * Non-fatal errors, count them and when the number is under ACL_REPORT_ERR_MAX_PER_JOB
648 * print the error message set by the lower level routine in jcr->errmsg.
650 if (jcr->acl_data->nr_errors < ACL_REPORT_ERR_MAX_PER_JOB) {
651 Jmsg(jcr, M_ERROR, 0, "%s", jcr->errmsg);
653 jcr->acl_data->nr_errors++;
662 * Save Extended Attributes when requested and available for all files not being a plugin.
665 if (ff_pkt->flags & FO_XATTR && !ff_pkt->cmd_plugin) {
666 switch (build_xattr_streams(jcr, ff_pkt)) {
667 case bxattr_exit_fatal:
669 case bxattr_exit_error:
671 * Non-fatal errors, count them and when the number is under XATTR_REPORT_ERR_MAX_PER_JOB
672 * print the error message set by the lower level routine in jcr->errmsg.
674 if (jcr->xattr_data->nr_errors < XATTR_REPORT_ERR_MAX_PER_JOB) {
675 Jmsg(jcr, M_ERROR, 0, "%s", jcr->errmsg);
677 jcr->xattr_data->nr_errors++;
685 /** Terminate the signing digest and send it to the Storage daemon */
686 if (signing_digest) {
689 if ((sig = crypto_sign_new(jcr)) == NULL) {
690 Jmsg(jcr, M_FATAL, 0, _("Failed to allocate memory for crypto signature.\n"));
694 if (!crypto_sign_add_signer(sig, signing_digest, jcr->crypto.pki_keypair)) {
695 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
699 /** Get signature size */
700 if (!crypto_sign_encode(sig, NULL, &size)) {
701 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
705 /** Grow the bsock buffer to fit our message if necessary */
706 if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
707 sd->msg = realloc_pool_memory(sd->msg, size);
710 /** Send our header */
711 sd->fsend("%ld %ld 0", jcr->JobFiles, STREAM_SIGNED_DIGEST);
712 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
714 /** Encode signature data */
715 if (!crypto_sign_encode(sig, (uint8_t *)sd->msg, &size)) {
716 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
722 sd->signal(BNET_EOD); /* end of checksum */
725 /** Terminate any digest and send it to Storage daemon */
729 sd->fsend("%ld %d 0", jcr->JobFiles, digest_stream);
730 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
732 size = CRYPTO_DIGEST_MAX_SIZE;
734 /** Grow the bsock buffer to fit our message if necessary */
735 if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
736 sd->msg = realloc_pool_memory(sd->msg, size);
739 if (!crypto_digest_finalize(digest, (uint8_t *)sd->msg, &size)) {
740 Jmsg(jcr, M_FATAL, 0, _("An error occurred finalizing signing the stream.\n"));
746 sd->signal(BNET_EOD); /* end of checksum */
750 rtnstat = jcr->is_canceled() ? 0 : 1; /* good return if not canceled */
753 if (jcr->is_incomplete()) {
756 if (ff_pkt->cmd_plugin && plugin_started) {
757 send_plugin_name(jcr, sd, false); /* signal end of plugin data */
760 crypto_digest_free(digest);
762 if (signing_digest) {
763 crypto_digest_free(signing_digest);
766 crypto_sign_free(sig);
772 * Send data read from an already open file descriptor.
774 * We return 1 on sucess and 0 on errors.
777 * We use ff_pkt->statp.st_size when FO_SPARSE to know when to stop
779 * Currently this is not a problem as the only other stream, resource forks,
780 * are not handled as sparse files.
782 static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest,
783 DIGEST *signing_digest)
785 BSOCK *sd = jcr->store_bsock;
786 uint64_t fileAddr = 0; /* file address */
788 int32_t rsize = jcr->buf_size; /* read buffer size */
790 CIPHER_CONTEXT *cipher_ctx = NULL; /* Quell bogus uninitialized warnings */
791 const uint8_t *cipher_input;
792 uint32_t cipher_input_len;
793 uint32_t cipher_block_size;
794 uint32_t encrypted_len;
795 #ifdef FD_NO_SEND_TEST
800 rbuf = sd->msg; /* read buffer */
801 wbuf = sd->msg; /* write buffer */
802 cipher_input = (uint8_t *)rbuf; /* encrypt uncompressed data */
804 Dmsg1(300, "Saving data, type=%d\n", ff_pkt->type);
807 uLong compress_len = 0;
808 uLong max_compress_len = 0;
809 const Bytef *cbuf = NULL;
812 if (ff_pkt->flags & FO_GZIP) {
813 if ((ff_pkt->flags & FO_SPARSE) || (ff_pkt->flags & FO_OFFSETS)) {
814 cbuf = (Bytef *)jcr->compress_buf + OFFSET_FADDR_SIZE;
815 max_compress_len = jcr->compress_buf_size - OFFSET_FADDR_SIZE;
817 cbuf = (Bytef *)jcr->compress_buf;
818 max_compress_len = jcr->compress_buf_size; /* set max length */
820 wbuf = jcr->compress_buf; /* compressed output here */
821 cipher_input = (uint8_t *)jcr->compress_buf; /* encrypt compressed data */
824 * Only change zlib parameters if there is no pending operation.
825 * This should never happen as deflatereset is called after each
829 if (((z_stream*)jcr->pZLIB_compress_workset)->total_in == 0) {
830 /** set gzip compression level - must be done per file */
831 if ((zstat=deflateParams((z_stream*)jcr->pZLIB_compress_workset,
832 ff_pkt->GZIP_level, Z_DEFAULT_STRATEGY)) != Z_OK) {
833 Jmsg(jcr, M_FATAL, 0, _("Compression deflateParams error: %d\n"), zstat);
834 set_jcr_job_status(jcr, JS_ErrorTerminated);
840 const uint32_t max_compress_len = 0;
843 if (ff_pkt->flags & FO_ENCRYPT) {
844 if ((ff_pkt->flags & FO_SPARSE) || (ff_pkt->flags & FO_OFFSETS)) {
845 Jmsg0(jcr, M_FATAL, 0, _("Encrypting sparse or offset data not supported.\n"));
848 /** Allocate the cipher context */
849 if ((cipher_ctx = crypto_cipher_new(jcr->crypto.pki_session, true,
850 &cipher_block_size)) == NULL) {
851 /* Shouldn't happen! */
852 Jmsg0(jcr, M_FATAL, 0, _("Failed to initialize encryption context.\n"));
857 * Grow the crypto buffer, if necessary.
858 * crypto_cipher_update() will buffer up to (cipher_block_size - 1).
859 * We grow crypto_buf to the maximum number of blocks that
860 * could be returned for the given read buffer size.
861 * (Using the larger of either rsize or max_compress_len)
863 jcr->crypto.crypto_buf = check_pool_memory_size(jcr->crypto.crypto_buf,
864 (MAX(rsize + (int)sizeof(uint32_t), (int32_t)max_compress_len) +
865 cipher_block_size - 1) / cipher_block_size * cipher_block_size);
867 wbuf = jcr->crypto.crypto_buf; /* Encrypted, possibly compressed output here. */
871 * Send Data header to Storage daemon
872 * <file-index> <stream> <info>
874 if (!sd->fsend("%ld %d 0", jcr->JobFiles, stream)) {
875 if (!jcr->is_job_canceled()) {
876 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
881 Dmsg1(300, ">stored: datahdr %s\n", sd->msg);
884 * Make space at beginning of buffer for fileAddr because this
885 * same buffer will be used for writing if compression is off.
887 if ((ff_pkt->flags & FO_SPARSE) || (ff_pkt->flags & FO_OFFSETS)) {
888 rbuf += OFFSET_FADDR_SIZE;
889 rsize -= OFFSET_FADDR_SIZE;
890 #ifdef HAVE_FREEBSD_OS
892 * To read FreeBSD partitions, the read size must be
895 rsize = (rsize/512) * 512;
899 /** a RAW device read on win32 only works if the buffer is a multiple of 512 */
901 if (S_ISBLK(ff_pkt->statp.st_mode))
902 rsize = (rsize/512) * 512;
908 while ((sd->msglen=(uint32_t)bread(&ff_pkt->bfd, rbuf, rsize)) > 0) {
910 /** Check for sparse blocks */
911 if (ff_pkt->flags & FO_SPARSE) {
913 bool allZeros = false;
914 if ((sd->msglen == rsize &&
915 fileAddr+sd->msglen < (uint64_t)ff_pkt->statp.st_size) ||
916 ((ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO) &&
917 (uint64_t)ff_pkt->statp.st_size == 0)) {
918 allZeros = is_buf_zero(rbuf, rsize);
921 /** Put file address as first data in buffer */
922 ser_begin(wbuf, OFFSET_FADDR_SIZE);
923 ser_uint64(fileAddr); /* store fileAddr in begin of buffer */
925 fileAddr += sd->msglen; /* update file address */
926 /** Skip block of all zeros */
928 continue; /* skip block of zeros */
930 } else if (ff_pkt->flags & FO_OFFSETS) {
932 ser_begin(wbuf, OFFSET_FADDR_SIZE);
933 ser_uint64(ff_pkt->bfd.offset); /* store offset in begin of buffer */
936 jcr->ReadBytes += sd->msglen; /* count bytes read */
938 /** Uncompressed cipher input length */
939 cipher_input_len = sd->msglen;
941 /** Update checksum if requested */
943 crypto_digest_update(digest, (uint8_t *)rbuf, sd->msglen);
946 /** Update signing digest if requested */
947 if (signing_digest) {
948 crypto_digest_update(signing_digest, (uint8_t *)rbuf, sd->msglen);
952 /** Do compression if turned on */
953 if (ff_pkt->flags & FO_GZIP && jcr->pZLIB_compress_workset) {
954 Dmsg3(400, "cbuf=0x%x rbuf=0x%x len=%u\n", cbuf, rbuf, sd->msglen);
956 ((z_stream*)jcr->pZLIB_compress_workset)->next_in = (Bytef *)rbuf;
957 ((z_stream*)jcr->pZLIB_compress_workset)->avail_in = sd->msglen;
958 ((z_stream*)jcr->pZLIB_compress_workset)->next_out = (Bytef *)cbuf;
959 ((z_stream*)jcr->pZLIB_compress_workset)->avail_out = max_compress_len;
961 if ((zstat=deflate((z_stream*)jcr->pZLIB_compress_workset, Z_FINISH)) != Z_STREAM_END) {
962 Jmsg(jcr, M_FATAL, 0, _("Compression deflate error: %d\n"), zstat);
963 set_jcr_job_status(jcr, JS_ErrorTerminated);
966 compress_len = ((z_stream*)jcr->pZLIB_compress_workset)->total_out;
967 /** reset zlib stream to be able to begin from scratch again */
968 if ((zstat=deflateReset((z_stream*)jcr->pZLIB_compress_workset)) != Z_OK) {
969 Jmsg(jcr, M_FATAL, 0, _("Compression deflateReset error: %d\n"), zstat);
970 set_jcr_job_status(jcr, JS_ErrorTerminated);
974 Dmsg2(400, "compressed len=%d uncompressed len=%d\n", compress_len,
977 sd->msglen = compress_len; /* set compressed length */
978 cipher_input_len = compress_len;
982 * Note, here we prepend the current record length to the beginning
983 * of the encrypted data. This is because both sparse and compression
984 * restore handling want records returned to them with exactly the
985 * same number of bytes that were processed in the backup handling.
986 * That is, both are block filters rather than a stream. When doing
987 * compression, the compression routines may buffer data, so that for
988 * any one record compressed, when it is decompressed the same size
989 * will not be obtained. Of course, the buffered data eventually comes
990 * out in subsequent crypto_cipher_update() calls or at least
991 * when crypto_cipher_finalize() is called. Unfortunately, this
992 * "feature" of encryption enormously complicates the restore code.
994 if (ff_pkt->flags & FO_ENCRYPT) {
995 uint32_t initial_len = 0;
998 if ((ff_pkt->flags & FO_SPARSE) || (ff_pkt->flags & FO_OFFSETS)) {
999 cipher_input_len += OFFSET_FADDR_SIZE;
1002 /** Encrypt the length of the input block */
1003 uint8_t packet_len[sizeof(uint32_t)];
1005 ser_begin(packet_len, sizeof(uint32_t));
1006 ser_uint32(cipher_input_len); /* store data len in begin of buffer */
1007 Dmsg1(20, "Encrypt len=%d\n", cipher_input_len);
1009 if (!crypto_cipher_update(cipher_ctx, packet_len, sizeof(packet_len),
1010 (uint8_t *)jcr->crypto.crypto_buf, &initial_len)) {
1011 /** Encryption failed. Shouldn't happen. */
1012 Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
1016 /** Encrypt the input block */
1017 if (crypto_cipher_update(cipher_ctx, cipher_input, cipher_input_len,
1018 (uint8_t *)&jcr->crypto.crypto_buf[initial_len], &encrypted_len)) {
1019 if ((initial_len + encrypted_len) == 0) {
1020 /** No full block of data available, read more data */
1023 Dmsg2(400, "encrypted len=%d unencrypted len=%d\n", encrypted_len,
1025 sd->msglen = initial_len + encrypted_len; /* set encrypted length */
1027 /** Encryption failed. Shouldn't happen. */
1028 Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
1033 /* Send the buffer to the Storage daemon */
1034 if ((ff_pkt->flags & FO_SPARSE) || (ff_pkt->flags & FO_OFFSETS)) {
1035 sd->msglen += OFFSET_FADDR_SIZE; /* include fileAddr in size */
1037 sd->msg = wbuf; /* set correct write buffer */
1039 if (!jcr->is_job_canceled()) {
1040 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1045 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
1047 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
1048 sd->msg = msgsave; /* restore read buffer */
1050 } /* end while read file data */
1052 if (sd->msglen < 0) { /* error */
1054 Jmsg(jcr, M_ERROR, 0, _("Read error on file %s. ERR=%s\n"),
1055 ff_pkt->fname, be.bstrerror(ff_pkt->bfd.berrno));
1056 if (jcr->JobErrors++ > 1000) { /* insanity check */
1057 Jmsg(jcr, M_FATAL, 0, _("Too many errors. JobErrors=%d.\n"), jcr->JobErrors);
1059 } else if (ff_pkt->flags & FO_ENCRYPT) {
1061 * For encryption, we must call finalize to push out any
1064 if (!crypto_cipher_finalize(cipher_ctx, (uint8_t *)jcr->crypto.crypto_buf,
1066 /* Padding failed. Shouldn't happen. */
1067 Jmsg(jcr, M_FATAL, 0, _("Encryption padding error\n"));
1071 /** Note, on SSL pre-0.9.7, there is always some output */
1072 if (encrypted_len > 0) {
1073 sd->msglen = encrypted_len; /* set encrypted length */
1074 sd->msg = jcr->crypto.crypto_buf; /* set correct write buffer */
1076 if (!jcr->is_job_canceled()) {
1077 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1082 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
1083 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
1084 sd->msg = msgsave; /* restore bnet buffer */
1088 if (!sd->signal(BNET_EOD)) { /* indicate end of file data */
1089 if (!jcr->is_job_canceled()) {
1090 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1096 /** Free the cipher context */
1098 crypto_cipher_free(cipher_ctx);
1103 /** Free the cipher context */
1105 crypto_cipher_free(cipher_ctx);
1108 sd->msg = msgsave; /* restore bnet buffer */
1113 bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream)
1115 BSOCK *sd = jcr->store_bsock;
1116 char attribs[MAXSTRING];
1117 char attribsExBuf[MAXSTRING];
1118 char *attribsEx = NULL;
1122 int hangup = get_hangup();
1123 #ifdef FD_NO_SEND_TEST
1127 Dmsg1(300, "encode_and_send_attrs fname=%s\n", ff_pkt->fname);
1128 /** Find what data stream we will use, then encode the attributes */
1129 if ((data_stream = select_data_stream(ff_pkt)) == STREAM_NONE) {
1130 /* This should not happen */
1131 Jmsg0(jcr, M_FATAL, 0, _("Invalid file flags, no supported data stream type.\n"));
1134 encode_stat(attribs, &ff_pkt->statp, ff_pkt->LinkFI, data_stream);
1136 /** Now possibly extend the attributes */
1137 if (ff_pkt->type == FT_RESTORE_FIRST) {
1138 attr_stream = STREAM_RESTORE_OBJECT;
1140 attribsEx = attribsExBuf;
1141 attr_stream = encode_attribsEx(jcr, attribsEx, ff_pkt);
1144 Dmsg3(300, "File %s\nattribs=%s\nattribsEx=%s\n", ff_pkt->fname, attribs, attribsEx);
1147 jcr->JobFiles++; /* increment number of files sent */
1148 ff_pkt->FileIndex = jcr->JobFiles; /* return FileIndex */
1149 pm_strcpy(jcr->last_fname, ff_pkt->fname);
1152 /* Debug code: check if we must hangup */
1153 if (hangup && (jcr->JobFiles > (uint32_t)hangup)) {
1154 jcr->setJobStatus(JS_Incomplete);
1155 Jmsg1(jcr, M_FATAL, 0, "Debug hangup requested after %d files.\n", hangup);
1160 * Send Attributes header to Storage daemon
1161 * <file-index> <stream> <info>
1163 if (!sd->fsend("%ld %d 0", jcr->JobFiles, attr_stream)) {
1164 if (!jcr->is_canceled()) {
1165 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1170 Dmsg1(300, ">stored: attrhdr %s\n", sd->msg);
1173 * Send file attributes to Storage daemon
1176 * Filename (full path)
1177 * Encoded attributes
1178 * Link name (if type==FT_LNK or FT_LNKSAVED)
1179 * Encoded extended-attributes (for Win32)
1181 * or send Restore Object to Storage daemon
1185 * Object_len (possibly compressed)
1186 * Object_full_len (not compressed)
1187 * Object_compression
1190 * Binary Object data
1192 * For a directory, link is the same as fname, but with trailing
1193 * slash. For a linked file, link is the link.
1195 if (ff_pkt->type != FT_DELETED) { /* already stripped */
1198 switch (ff_pkt->type) {
1201 Dmsg2(300, "Link %s to %s\n", ff_pkt->fname, ff_pkt->link);
1202 stat = sd->fsend("%ld %d %s%c%s%c%s%c%s%c%u%c", jcr->JobFiles,
1203 ff_pkt->type, ff_pkt->fname, 0, attribs, 0,
1204 ff_pkt->link, 0, attribsEx, 0, ff_pkt->delta_seq, 0);
1209 /* Here link is the canonical filename (i.e. with trailing slash) */
1210 stat = sd->fsend("%ld %d %s%c%s%c%c%s%c%u%c", jcr->JobFiles,
1211 ff_pkt->type, ff_pkt->link, 0, attribs, 0, 0,
1212 attribsEx, 0, ff_pkt->delta_seq, 0);
1214 case FT_RESTORE_FIRST:
1215 comp_len = ff_pkt->object_len;
1216 ff_pkt->object_compression = 0;
1217 if (ff_pkt->object_len > 1000) {
1218 /* Big object, compress it */
1220 comp_len = ff_pkt->object_len + 1000;
1221 POOLMEM *comp_obj = get_memory(comp_len);
1222 stat = Zdeflate(ff_pkt->object, ff_pkt->object_len, comp_obj, comp_len);
1223 if (comp_len < ff_pkt->object_len) {
1224 ff_pkt->object = comp_obj;
1225 ff_pkt->object_compression = 1; /* zlib level 9 compression */
1227 /* Uncompressed object smaller, use it */
1228 comp_len = ff_pkt->object_len;
1230 Dmsg2(100, "Object compressed from %d to %d bytes\n", ff_pkt->object_len, comp_len);
1232 sd->msglen = Mmsg(sd->msg, "%d %d %d %d %d %d %s%c%s%c",
1233 jcr->JobFiles, ff_pkt->type, ff_pkt->object_index,
1234 comp_len, ff_pkt->object_len, ff_pkt->object_compression,
1235 ff_pkt->fname, 0, ff_pkt->object_name, 0);
1236 sd->msg = check_pool_memory_size(sd->msg, sd->msglen + comp_len + 2);
1237 memcpy(sd->msg + sd->msglen, ff_pkt->object, comp_len);
1238 /* Note we send one extra byte so Dir can store zero after object */
1239 sd->msglen += comp_len + 1;
1241 if (ff_pkt->object_compression) {
1242 free_and_null_pool_memory(ff_pkt->object);
1246 stat = sd->fsend("%ld %d %s%c%s%c%c%s%c%d%c", jcr->JobFiles,
1247 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, 0, attribsEx, 0,
1248 ff_pkt->delta_seq, 0);
1251 stat = sd->fsend("%ld %d %s%c%s%c%c%s%c%u%c", jcr->JobFiles,
1252 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, 0,
1253 attribsEx, 0, ff_pkt->delta_seq, 0);
1256 if (ff_pkt->type != FT_DELETED) {
1257 unstrip_path(ff_pkt);
1260 Dmsg2(300, ">stored: attr len=%d: %s\n", sd->msglen, sd->msg);
1261 if (!stat && !jcr->is_job_canceled()) {
1262 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1265 sd->signal(BNET_EOD); /* indicate end of attributes data */
1270 * Do in place strip of path
1272 static bool do_strip(int count, char *in)
1278 /** Copy to first path separator -- Win32 might have c: ... */
1279 while (*in && !IsPathSeparator(*in)) {
1283 numsep++; /* one separator seen */
1284 for (stripped=0; stripped<count && *in; stripped++) {
1285 while (*in && !IsPathSeparator(*in)) {
1286 in++; /* skip chars */
1289 numsep++; /* count separators seen */
1290 in++; /* skip separator */
1294 while (*in) { /* copy to end */
1295 if (IsPathSeparator(*in)) {
1301 Dmsg4(500, "stripped=%d count=%d numsep=%d sep>count=%d\n",
1302 stripped, count, numsep, numsep>count);
1303 return stripped==count && numsep>count;
1307 * If requested strip leading components of the path so that we can
1308 * save file as if it came from a subdirectory. This is most useful
1309 * for dealing with snapshots, by removing the snapshot directory, or
1310 * in handling vendor migrations where files have been restored with
1311 * a vendor product into a subdirectory.
1313 void strip_path(FF_PKT *ff_pkt)
1315 if (!(ff_pkt->flags & FO_STRIPPATH) || ff_pkt->strip_path <= 0) {
1316 Dmsg1(200, "No strip for %s\n", ff_pkt->fname);
1319 if (!ff_pkt->fname_save) {
1320 ff_pkt->fname_save = get_pool_memory(PM_FNAME);
1321 ff_pkt->link_save = get_pool_memory(PM_FNAME);
1323 pm_strcpy(ff_pkt->fname_save, ff_pkt->fname);
1324 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1325 pm_strcpy(ff_pkt->link_save, ff_pkt->link);
1326 Dmsg2(500, "strcpy link_save=%d link=%d\n", strlen(ff_pkt->link_save),
1327 strlen(ff_pkt->link));
1328 sm_check(__FILE__, __LINE__, true);
1332 * Strip path. If it doesn't succeed put it back. If
1333 * it does, and there is a different link string,
1334 * attempt to strip the link. If it fails, back them
1336 * Do not strip symlinks.
1337 * I.e. if either stripping fails don't strip anything.
1339 if (!do_strip(ff_pkt->strip_path, ff_pkt->fname)) {
1340 unstrip_path(ff_pkt);
1343 /** Strip links but not symlinks */
1344 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1345 if (!do_strip(ff_pkt->strip_path, ff_pkt->link)) {
1346 unstrip_path(ff_pkt);
1351 Dmsg3(100, "fname=%s stripped=%s link=%s\n", ff_pkt->fname_save, ff_pkt->fname,
1355 void unstrip_path(FF_PKT *ff_pkt)
1357 if (!(ff_pkt->flags & FO_STRIPPATH) || ff_pkt->strip_path <= 0) {
1360 strcpy(ff_pkt->fname, ff_pkt->fname_save);
1361 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1362 Dmsg2(500, "strcpy link=%s link_save=%s\n", ff_pkt->link,
1364 strcpy(ff_pkt->link, ff_pkt->link_save);
1365 Dmsg2(500, "strcpy link=%d link_save=%d\n", strlen(ff_pkt->link),
1366 strlen(ff_pkt->link_save));
1367 sm_check(__FILE__, __LINE__, true);
1371 static void close_vss_backup_session(JCR *jcr)
1373 #if defined(WIN32_VSS)
1374 /* STOP VSS ON WIN32 */
1375 /* tell vss to close the backup session */
1377 if (g_pVSSClient->CloseBackup()) {
1378 /* inform user about writer states */
1379 for (int i=0; i<(int)g_pVSSClient->GetWriterCount(); i++) {
1380 int msg_type = M_INFO;
1381 if (g_pVSSClient->GetWriterState(i) < 1) {
1382 msg_type = M_WARNING;
1385 Jmsg(jcr, msg_type, 0, _("VSS Writer (BackupComplete): %s\n"), g_pVSSClient->GetWriterInfo(i));
1388 WCHAR *metadata = g_pVSSClient->GetMetadata();
1390 FF_PKT *ff_pkt = jcr->ff;
1391 ff_pkt->fname = (char *)"job";
1392 ff_pkt->type = FT_RESTORE_FIRST;
1394 ff_pkt->object_name = (char *)"job_metadata.xml";
1395 ff_pkt->object = (char *)metadata;
1396 ff_pkt->object_len = (wcslen(metadata) + 1) * sizeof(WCHAR);
1397 ff_pkt->object_index = (int)time(NULL);
1398 save_file(jcr, ff_pkt, true);