2 Bacula® - The Network Backup Solution
4 Copyright (C) 2000-2010 Free Software Foundation Europe e.V.
6 The main author of Bacula is Kern Sibbald, with contributions from
7 many others, a complete list can be found in the file AUTHORS.
8 This program is Free Software; you can redistribute it and/or
9 modify it under the terms of version two of the GNU General Public
10 License as published by the Free Software Foundation and included
13 This program is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
23 Bacula® is a registered trademark of Kern Sibbald.
24 The licensor of Bacula is the Free Software Foundation Europe
25 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
26 Switzerland, email:ftf@fsfeurope.org.
29 * Bacula File Daemon backup.c send file attributes and data
30 * to the Storage daemon.
32 * Kern Sibbald, March MM
40 const bool have_darwin_os = true;
42 const bool have_darwin_os = false;
46 const bool have_acl = true;
48 const bool have_acl = false;
51 #if defined(HAVE_XATTR)
52 const bool have_xattr = true;
54 const bool have_xattr = false;
57 /* Forward referenced functions */
58 int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level);
59 static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, DIGEST *signature_digest);
60 bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream);
61 static bool crypto_session_start(JCR *jcr);
62 static void crypto_session_end(JCR *jcr);
63 static bool crypto_session_send(JCR *jcr, BSOCK *sd);
64 static void close_vss_backup_session(JCR *jcr);
67 * Find all the requested files and send them
68 * to the Storage daemon.
70 * Note, we normally carry on a one-way
71 * conversation from this point on with the SD, simply blasting
72 * data to him. To properly know what is going on, we
73 * also run a "heartbeat" monitor which reads the socket and
74 * reacts accordingly (at the moment it has nothing to do
75 * except echo the heartbeat to the Director).
78 bool blast_data_to_storage_daemon(JCR *jcr, char *addr)
82 // TODO landonf: Allow user to specify encryption algorithm
84 sd = jcr->store_bsock;
86 set_jcr_job_status(jcr, JS_Running);
88 Dmsg1(300, "bfiled: opened data connection %d to stored\n", sd->m_fd);
91 CLIENT *client = (CLIENT *)GetNextRes(R_CLIENT, NULL);
95 buf_size = client->max_network_buffer_size;
97 buf_size = 0; /* use default */
99 if (!sd->set_buffer_size(buf_size, BNET_SETBUF_WRITE)) {
100 set_jcr_job_status(jcr, JS_ErrorTerminated);
101 Jmsg(jcr, M_FATAL, 0, _("Cannot set buffer size FD->SD.\n"));
105 jcr->buf_size = sd->msglen;
107 * Adjust for compression so that output buffer is
108 * 12 bytes + 0.1% larger than input buffer plus 18 bytes.
109 * This gives a bit extra plus room for the sparse addr if any.
110 * Note, we adjust the read size to be smaller so that the
111 * same output buffer can be used without growing it.
113 * The zlib compression workset is initialized here to minimize
114 * the "per file" load. The jcr member is only set, if the init
117 jcr->compress_buf_size = jcr->buf_size + ((jcr->buf_size+999) / 1000) + 30;
118 jcr->compress_buf = get_memory(jcr->compress_buf_size);
121 z_stream *pZlibStream = (z_stream*)malloc(sizeof(z_stream));
123 pZlibStream->zalloc = Z_NULL;
124 pZlibStream->zfree = Z_NULL;
125 pZlibStream->opaque = Z_NULL;
126 pZlibStream->state = Z_NULL;
128 if (deflateInit(pZlibStream, Z_DEFAULT_COMPRESSION) == Z_OK) {
129 jcr->pZLIB_compress_workset = pZlibStream;
136 if (!crypto_session_start(jcr)) {
140 set_find_options((FF_PKT *)jcr->ff, jcr->incremental, jcr->mtime);
142 /** in accurate mode, we overwrite the find_one check function */
144 set_find_changed_function((FF_PKT *)jcr->ff, accurate_check_file);
147 start_heartbeat_monitor(jcr);
150 jcr->acl_data = (acl_data_t *)malloc(sizeof(acl_data_t));
151 memset((caddr_t)jcr->acl_data, 0, sizeof(acl_data_t));
152 jcr->acl_data->content = get_pool_memory(PM_MESSAGE);
156 jcr->xattr_data = (xattr_data_t *)malloc(sizeof(xattr_data_t));
157 memset((caddr_t)jcr->xattr_data, 0, sizeof(xattr_data_t));
158 jcr->xattr_data->content = get_pool_memory(PM_MESSAGE);
161 /** Subroutine save_file() is called for each file */
162 if (!find_files(jcr, (FF_PKT *)jcr->ff, save_file, plugin_save)) {
163 ok = false; /* error */
164 set_jcr_job_status(jcr, JS_ErrorTerminated);
167 if (have_acl && jcr->acl_data->nr_errors > 0) {
168 Jmsg(jcr, M_ERROR, 0, _("Encountered %ld acl errors while doing backup\n"),
169 jcr->acl_data->nr_errors);
171 if (have_xattr && jcr->xattr_data->nr_errors > 0) {
172 Jmsg(jcr, M_ERROR, 0, _("Encountered %ld xattr errors while doing backup\n"),
173 jcr->xattr_data->nr_errors);
176 close_vss_backup_session(jcr);
178 accurate_finish(jcr); /* send deleted or base file list to SD */
180 stop_heartbeat_monitor(jcr);
182 sd->signal(BNET_EOD); /* end of sending data */
184 if (have_acl && jcr->acl_data) {
185 free_pool_memory(jcr->acl_data->content);
187 jcr->acl_data = NULL;
189 if (have_xattr && jcr->xattr_data) {
190 free_pool_memory(jcr->xattr_data->content);
191 free(jcr->xattr_data);
192 jcr->xattr_data = NULL;
198 if (jcr->compress_buf) {
199 free_pool_memory(jcr->compress_buf);
200 jcr->compress_buf = NULL;
202 if (jcr->pZLIB_compress_workset) {
203 /* Free the zlib stream */
205 deflateEnd((z_stream *)jcr->pZLIB_compress_workset);
207 free (jcr->pZLIB_compress_workset);
208 jcr->pZLIB_compress_workset = NULL;
210 crypto_session_end(jcr);
213 Dmsg1(100, "end blast_data ok=%d\n", ok);
217 static bool crypto_session_start(JCR *jcr)
219 crypto_cipher_t cipher = CRYPTO_CIPHER_AES_128_CBC;
222 * Create encryption session data and a cached, DER-encoded session data
223 * structure. We use a single session key for each backup, so we'll encode
224 * the session data only once.
226 if (jcr->crypto.pki_encrypt) {
229 /** Create per-job session encryption context */
230 jcr->crypto.pki_session = crypto_session_new(cipher, jcr->crypto.pki_recipients);
232 /** Get the session data size */
233 if (!crypto_session_encode(jcr->crypto.pki_session, (uint8_t *)0, &size)) {
234 Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n"));
238 /** Allocate buffer */
239 jcr->crypto.pki_session_encoded = get_memory(size);
241 /** Encode session data */
242 if (!crypto_session_encode(jcr->crypto.pki_session, (uint8_t *)jcr->crypto.pki_session_encoded, &size)) {
243 Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n"));
247 /** ... and store the encoded size */
248 jcr->crypto.pki_session_encoded_size = size;
250 /** Allocate the encryption/decryption buffer */
251 jcr->crypto.crypto_buf = get_memory(CRYPTO_CIPHER_MAX_BLOCK_SIZE);
256 static void crypto_session_end(JCR *jcr)
258 if (jcr->crypto.crypto_buf) {
259 free_pool_memory(jcr->crypto.crypto_buf);
260 jcr->crypto.crypto_buf = NULL;
262 if (jcr->crypto.pki_session) {
263 crypto_session_free(jcr->crypto.pki_session);
265 if (jcr->crypto.pki_session_encoded) {
266 free_pool_memory(jcr->crypto.pki_session_encoded);
267 jcr->crypto.pki_session_encoded = NULL;
271 static bool crypto_session_send(JCR *jcr, BSOCK *sd)
275 /** Send our header */
276 Dmsg2(100, "Send hdr fi=%ld stream=%d\n", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA);
277 sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA);
280 sd->msg = jcr->crypto.pki_session_encoded;
281 sd->msglen = jcr->crypto.pki_session_encoded_size;
282 jcr->JobBytes += sd->msglen;
284 Dmsg1(100, "Send data len=%d\n", sd->msglen);
287 sd->signal(BNET_EOD);
293 * Called here by find() for each file included.
294 * This is a callback. The original is find_files() above.
296 * Send the file and its data to the Storage daemon.
300 * -1 to ignore file/directory (not used here)
302 int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level)
304 bool do_read = false;
305 bool plugin_started = false;
306 int stat, data_stream;
308 DIGEST *digest = NULL;
309 DIGEST *signing_digest = NULL;
310 int digest_stream = STREAM_NONE;
311 SIGNATURE *sig = NULL;
312 bool has_file_data = false;
313 // TODO landonf: Allow the user to specify the digest algorithm
315 crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA256;
317 crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA1;
319 BSOCK *sd = jcr->store_bsock;
321 if (jcr->is_job_canceled()) {
325 jcr->num_files_examined++; /* bump total file count */
327 switch (ff_pkt->type) {
328 case FT_LNKSAVED: /* Hard linked, file already saved */
329 Dmsg2(130, "FT_LNKSAVED hard link: %s => %s\n", ff_pkt->fname, ff_pkt->link);
332 Dmsg1(130, "FT_REGE saving: %s\n", ff_pkt->fname);
333 has_file_data = true;
336 Dmsg1(130, "FT_REG saving: %s\n", ff_pkt->fname);
337 has_file_data = true;
340 Dmsg2(130, "FT_LNK saving: %s -> %s\n", ff_pkt->fname, ff_pkt->link);
342 case FT_RESTORE_FIRST:
343 Dmsg1(100, "FT_RESTORE_FIRST saving: %s\n", ff_pkt->fname);
346 jcr->num_files_examined--; /* correct file count */
347 return 1; /* not used */
349 Jmsg(jcr, M_INFO, 1, _(" Recursion turned off. Will not descend from %s into %s\n"),
350 ff_pkt->top_fname, ff_pkt->fname);
351 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
354 /* Suppress message for /dev filesystems */
355 if (!is_in_fileset(ff_pkt)) {
356 Jmsg(jcr, M_INFO, 1, _(" %s is a different filesystem. Will not descend from %s into %s\n"),
357 ff_pkt->fname, ff_pkt->top_fname, ff_pkt->fname);
359 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
362 Jmsg(jcr, M_INFO, 1, _(" Disallowed filesystem. Will not descend from %s into %s\n"),
363 ff_pkt->top_fname, ff_pkt->fname);
364 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
367 Jmsg(jcr, M_INFO, 1, _(" Disallowed drive type. Will not descend into %s\n"),
372 Dmsg1(130, "FT_DIREND: %s\n", ff_pkt->link);
375 Dmsg1(130, "FT_SPEC saving: %s\n", ff_pkt->fname);
376 if (S_ISSOCK(ff_pkt->statp.st_mode)) {
377 Jmsg(jcr, M_SKIPPED, 1, _(" Socket file skipped: %s\n"), ff_pkt->fname);
382 Dmsg1(130, "FT_RAW saving: %s\n", ff_pkt->fname);
383 has_file_data = true;
386 Dmsg1(130, "FT_FIFO saving: %s\n", ff_pkt->fname);
390 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not access \"%s\": ERR=%s\n"), ff_pkt->fname,
391 be.bstrerror(ff_pkt->ff_errno));
397 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not follow link \"%s\": ERR=%s\n"),
398 ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
404 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not stat \"%s\": ERR=%s\n"), ff_pkt->fname,
405 be.bstrerror(ff_pkt->ff_errno));
411 Jmsg(jcr, M_SKIPPED, 1, _(" Unchanged file skipped: %s\n"), ff_pkt->fname);
414 Jmsg(jcr, M_NOTSAVED, 0, _(" Archive file not saved: %s\n"), ff_pkt->fname);
418 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not open directory \"%s\": ERR=%s\n"),
419 ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
424 Jmsg(jcr, M_NOTSAVED, 0, _(" Unknown file type %d; not saved: %s\n"),
425 ff_pkt->type, ff_pkt->fname);
430 Dmsg1(130, "bfiled: sending %s to stored\n", ff_pkt->fname);
432 /** Digests and encryption are only useful if there's file data */
435 * Setup for digest handling. If this fails, the digest will be set to NULL
436 * and not used. Note, the digest (file hash) can be any one of the four
439 * The signing digest is a single algorithm depending on
440 * whether or not we have SHA2.
441 * ****FIXME**** the signing algoritm should really be
442 * determined a different way!!!!!! What happens if
443 * sha2 was available during backup but not restore?
445 if (ff_pkt->flags & FO_MD5) {
446 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_MD5);
447 digest_stream = STREAM_MD5_DIGEST;
449 } else if (ff_pkt->flags & FO_SHA1) {
450 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA1);
451 digest_stream = STREAM_SHA1_DIGEST;
453 } else if (ff_pkt->flags & FO_SHA256) {
454 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA256);
455 digest_stream = STREAM_SHA256_DIGEST;
457 } else if (ff_pkt->flags & FO_SHA512) {
458 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA512);
459 digest_stream = STREAM_SHA512_DIGEST;
462 /** Did digest initialization fail? */
463 if (digest_stream != STREAM_NONE && digest == NULL) {
464 Jmsg(jcr, M_WARNING, 0, _("%s digest initialization failed\n"),
465 stream_to_ascii(digest_stream));
469 * Set up signature digest handling. If this fails, the signature digest
470 * will be set to NULL and not used.
472 /* TODO landonf: We should really only calculate the digest once, for
473 * both verification and signing.
475 if (jcr->crypto.pki_sign) {
476 signing_digest = crypto_digest_new(jcr, signing_algorithm);
478 /** Full-stop if a failure occurred initializing the signature digest */
479 if (signing_digest == NULL) {
480 Jmsg(jcr, M_NOTSAVED, 0, _("%s signature digest initialization failed\n"),
481 stream_to_ascii(signing_algorithm));
487 /** Enable encryption */
488 if (jcr->crypto.pki_encrypt) {
489 ff_pkt->flags |= FO_ENCRYPT;
493 /** Initialize the file descriptor we use for data and other streams. */
495 if (ff_pkt->flags & FO_PORTABLE) {
496 set_portable_backup(&ff_pkt->bfd); /* disable Win32 BackupRead() */
498 if (ff_pkt->cmd_plugin) {
499 /* Tell bfile that it needs to call plugin */
500 if (!set_cmd_plugin(&ff_pkt->bfd, jcr)) {
503 send_plugin_name(jcr, sd, true); /* signal start of plugin data */
504 plugin_started = true;
507 /** Send attributes -- must be done after binit() */
508 if (!encode_and_send_attributes(jcr, ff_pkt, data_stream)) {
511 /** Meta data only for restore object */
512 if (ff_pkt->type == FT_RESTORE_FIRST) {
516 /** Set up the encryption context and send the session data to the SD */
517 if (has_file_data && jcr->crypto.pki_encrypt) {
518 if (!crypto_session_send(jcr, sd)) {
524 * Open any file with data that we intend to save, then save it.
526 * Note, if is_win32_backup, we must open the Directory so that
527 * the BackupRead will save its permissions and ownership streams.
529 if (ff_pkt->type != FT_LNKSAVED && S_ISREG(ff_pkt->statp.st_mode)) {
531 do_read = !is_portable_backup(&ff_pkt->bfd) || ff_pkt->statp.st_size > 0;
533 do_read = ff_pkt->statp.st_size > 0;
535 } else if (ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO ||
536 ff_pkt->type == FT_REPARSE ||
537 (!is_portable_backup(&ff_pkt->bfd) && ff_pkt->type == FT_DIREND)) {
541 if (ff_pkt->cmd_plugin) {
545 Dmsg2(150, "type=%d do_read=%d\n", ff_pkt->type, do_read);
549 if (ff_pkt->type == FT_FIFO) {
550 tid = start_thread_timer(jcr, pthread_self(), 60);
554 int noatime = ff_pkt->flags & FO_NOATIME ? O_NOATIME : 0;
555 ff_pkt->bfd.reparse_point = ff_pkt->type == FT_REPARSE;
556 if (bopen(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY | noatime, 0) < 0) {
557 ff_pkt->ff_errno = errno;
559 Jmsg(jcr, M_NOTSAVED, 0, _(" Cannot open \"%s\": ERR=%s.\n"), ff_pkt->fname,
563 stop_thread_timer(tid);
569 stop_thread_timer(tid);
573 stat = send_data(jcr, data_stream, ff_pkt, digest, signing_digest);
575 if (ff_pkt->flags & FO_CHKCHANGES) {
576 has_file_changed(jcr, ff_pkt);
579 bclose(&ff_pkt->bfd);
586 if (have_darwin_os) {
587 /** Regular files can have resource forks and Finder Info */
588 if (ff_pkt->type != FT_LNKSAVED && (S_ISREG(ff_pkt->statp.st_mode) &&
589 ff_pkt->flags & FO_HFSPLUS)) {
590 if (ff_pkt->hfsinfo.rsrclength > 0) {
593 if (!bopen_rsrc(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0) < 0) {
594 ff_pkt->ff_errno = errno;
596 Jmsg(jcr, M_NOTSAVED, -1, _(" Cannot open resource fork for \"%s\": ERR=%s.\n"),
597 ff_pkt->fname, be.bstrerror());
599 if (is_bopen(&ff_pkt->bfd)) {
600 bclose(&ff_pkt->bfd);
604 flags = ff_pkt->flags;
605 ff_pkt->flags &= ~(FO_GZIP|FO_SPARSE);
606 if (flags & FO_ENCRYPT) {
607 rsrc_stream = STREAM_ENCRYPTED_MACOS_FORK_DATA;
609 rsrc_stream = STREAM_MACOS_FORK_DATA;
611 stat = send_data(jcr, rsrc_stream, ff_pkt, digest, signing_digest);
612 ff_pkt->flags = flags;
613 bclose(&ff_pkt->bfd);
619 Dmsg1(300, "Saving Finder Info for \"%s\"\n", ff_pkt->fname);
620 sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_HFSPLUS_ATTRIBUTES);
621 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
622 pm_memcpy(sd->msg, ff_pkt->hfsinfo.fndrinfo, 32);
625 crypto_digest_update(digest, (uint8_t *)sd->msg, sd->msglen);
627 if (signing_digest) {
628 crypto_digest_update(signing_digest, (uint8_t *)sd->msg, sd->msglen);
631 sd->signal(BNET_EOD);
636 * Save ACLs when requested and available for anything not being a symlink and not being a plugin.
639 if (ff_pkt->flags & FO_ACL && ff_pkt->type != FT_LNK && !ff_pkt->cmd_plugin) {
640 switch (build_acl_streams(jcr, ff_pkt)) {
641 case bacl_exit_fatal:
643 case bacl_exit_error:
645 * Non-fatal errors, count them and when the number is under ACL_REPORT_ERR_MAX_PER_JOB
646 * print the error message set by the lower level routine in jcr->errmsg.
648 if (jcr->acl_data->nr_errors < ACL_REPORT_ERR_MAX_PER_JOB) {
649 Jmsg(jcr, M_ERROR, 0, "%s", jcr->errmsg);
651 jcr->acl_data->nr_errors++;
660 * Save Extended Attributes when requested and available for all files not being a plugin.
663 if (ff_pkt->flags & FO_XATTR && !ff_pkt->cmd_plugin) {
664 switch (build_xattr_streams(jcr, ff_pkt)) {
665 case bxattr_exit_fatal:
667 case bxattr_exit_error:
669 * Non-fatal errors, count them and when the number is under XATTR_REPORT_ERR_MAX_PER_JOB
670 * print the error message set by the lower level routine in jcr->errmsg.
672 if (jcr->xattr_data->nr_errors < XATTR_REPORT_ERR_MAX_PER_JOB) {
673 Jmsg(jcr, M_ERROR, 0, "%s", jcr->errmsg);
675 jcr->xattr_data->nr_errors++;
683 /** Terminate the signing digest and send it to the Storage daemon */
684 if (signing_digest) {
687 if ((sig = crypto_sign_new(jcr)) == NULL) {
688 Jmsg(jcr, M_FATAL, 0, _("Failed to allocate memory for crypto signature.\n"));
692 if (!crypto_sign_add_signer(sig, signing_digest, jcr->crypto.pki_keypair)) {
693 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
697 /** Get signature size */
698 if (!crypto_sign_encode(sig, NULL, &size)) {
699 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
703 /** Grow the bsock buffer to fit our message if necessary */
704 if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
705 sd->msg = realloc_pool_memory(sd->msg, size);
708 /** Send our header */
709 sd->fsend("%ld %ld 0", jcr->JobFiles, STREAM_SIGNED_DIGEST);
710 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
712 /** Encode signature data */
713 if (!crypto_sign_encode(sig, (uint8_t *)sd->msg, &size)) {
714 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
720 sd->signal(BNET_EOD); /* end of checksum */
723 /** Terminate any digest and send it to Storage daemon */
727 sd->fsend("%ld %d 0", jcr->JobFiles, digest_stream);
728 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
730 size = CRYPTO_DIGEST_MAX_SIZE;
732 /** Grow the bsock buffer to fit our message if necessary */
733 if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
734 sd->msg = realloc_pool_memory(sd->msg, size);
737 if (!crypto_digest_finalize(digest, (uint8_t *)sd->msg, &size)) {
738 Jmsg(jcr, M_FATAL, 0, _("An error occurred finalizing signing the stream.\n"));
744 sd->signal(BNET_EOD); /* end of checksum */
748 rtnstat = 1; /* good return */
751 if (ff_pkt->cmd_plugin && plugin_started) {
752 send_plugin_name(jcr, sd, false); /* signal end of plugin data */
755 crypto_digest_free(digest);
757 if (signing_digest) {
758 crypto_digest_free(signing_digest);
761 crypto_sign_free(sig);
767 * Send data read from an already open file descriptor.
769 * We return 1 on sucess and 0 on errors.
772 * We use ff_pkt->statp.st_size when FO_SPARSE to know when to stop
774 * Currently this is not a problem as the only other stream, resource forks,
775 * are not handled as sparse files.
777 static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest,
778 DIGEST *signing_digest)
780 BSOCK *sd = jcr->store_bsock;
781 uint64_t fileAddr = 0; /* file address */
783 int32_t rsize = jcr->buf_size; /* read buffer size */
785 CIPHER_CONTEXT *cipher_ctx = NULL; /* Quell bogus uninitialized warnings */
786 const uint8_t *cipher_input;
787 uint32_t cipher_input_len;
788 uint32_t cipher_block_size;
789 uint32_t encrypted_len;
790 #ifdef FD_NO_SEND_TEST
795 rbuf = sd->msg; /* read buffer */
796 wbuf = sd->msg; /* write buffer */
797 cipher_input = (uint8_t *)rbuf; /* encrypt uncompressed data */
799 Dmsg1(300, "Saving data, type=%d\n", ff_pkt->type);
802 uLong compress_len = 0;
803 uLong max_compress_len = 0;
804 const Bytef *cbuf = NULL;
807 if (ff_pkt->flags & FO_GZIP) {
808 if (ff_pkt->flags & FO_SPARSE) {
809 cbuf = (Bytef *)jcr->compress_buf + SPARSE_FADDR_SIZE;
810 max_compress_len = jcr->compress_buf_size - SPARSE_FADDR_SIZE;
812 cbuf = (Bytef *)jcr->compress_buf;
813 max_compress_len = jcr->compress_buf_size; /* set max length */
815 wbuf = jcr->compress_buf; /* compressed output here */
816 cipher_input = (uint8_t *)jcr->compress_buf; /* encrypt compressed data */
819 * Only change zlib parameters if there is no pending operation.
820 * This should never happen as deflatereset is called after each
824 if (((z_stream*)jcr->pZLIB_compress_workset)->total_in == 0) {
825 /** set gzip compression level - must be done per file */
826 if ((zstat=deflateParams((z_stream*)jcr->pZLIB_compress_workset,
827 ff_pkt->GZIP_level, Z_DEFAULT_STRATEGY)) != Z_OK) {
828 Jmsg(jcr, M_FATAL, 0, _("Compression deflateParams error: %d\n"), zstat);
829 set_jcr_job_status(jcr, JS_ErrorTerminated);
835 const uint32_t max_compress_len = 0;
838 if (ff_pkt->flags & FO_ENCRYPT) {
839 if (ff_pkt->flags & FO_SPARSE) {
840 Jmsg0(jcr, M_FATAL, 0, _("Encrypting sparse data not supported.\n"));
843 /** Allocate the cipher context */
844 if ((cipher_ctx = crypto_cipher_new(jcr->crypto.pki_session, true,
845 &cipher_block_size)) == NULL) {
846 /* Shouldn't happen! */
847 Jmsg0(jcr, M_FATAL, 0, _("Failed to initialize encryption context.\n"));
852 * Grow the crypto buffer, if necessary.
853 * crypto_cipher_update() will buffer up to (cipher_block_size - 1).
854 * We grow crypto_buf to the maximum number of blocks that
855 * could be returned for the given read buffer size.
856 * (Using the larger of either rsize or max_compress_len)
858 jcr->crypto.crypto_buf = check_pool_memory_size(jcr->crypto.crypto_buf,
859 (MAX(rsize + (int)sizeof(uint32_t), (int32_t)max_compress_len) +
860 cipher_block_size - 1) / cipher_block_size * cipher_block_size);
862 wbuf = jcr->crypto.crypto_buf; /* Encrypted, possibly compressed output here. */
866 * Send Data header to Storage daemon
867 * <file-index> <stream> <info>
869 if (!sd->fsend("%ld %d 0", jcr->JobFiles, stream)) {
870 if (!jcr->is_job_canceled()) {
871 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
876 Dmsg1(300, ">stored: datahdr %s\n", sd->msg);
879 * Make space at beginning of buffer for fileAddr because this
880 * same buffer will be used for writing if compression is off.
882 if (ff_pkt->flags & FO_SPARSE) {
883 rbuf += SPARSE_FADDR_SIZE;
884 rsize -= SPARSE_FADDR_SIZE;
885 #ifdef HAVE_FREEBSD_OS
887 * To read FreeBSD partitions, the read size must be
890 rsize = (rsize/512) * 512;
894 /** a RAW device read on win32 only works if the buffer is a multiple of 512 */
896 if (S_ISBLK(ff_pkt->statp.st_mode))
897 rsize = (rsize/512) * 512;
903 while ((sd->msglen=(uint32_t)bread(&ff_pkt->bfd, rbuf, rsize)) > 0) {
905 /** Check for sparse blocks */
906 if (ff_pkt->flags & FO_SPARSE) {
908 bool allZeros = false;
909 if ((sd->msglen == rsize &&
910 fileAddr+sd->msglen < (uint64_t)ff_pkt->statp.st_size) ||
911 ((ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO) &&
912 (uint64_t)ff_pkt->statp.st_size == 0)) {
913 allZeros = is_buf_zero(rbuf, rsize);
916 /** Put file address as first data in buffer */
917 ser_begin(wbuf, SPARSE_FADDR_SIZE);
918 ser_uint64(fileAddr); /* store fileAddr in begin of buffer */
920 fileAddr += sd->msglen; /* update file address */
921 /** Skip block of all zeros */
923 continue; /* skip block of zeros */
927 jcr->ReadBytes += sd->msglen; /* count bytes read */
929 /** Uncompressed cipher input length */
930 cipher_input_len = sd->msglen;
932 /** Update checksum if requested */
934 crypto_digest_update(digest, (uint8_t *)rbuf, sd->msglen);
937 /** Update signing digest if requested */
938 if (signing_digest) {
939 crypto_digest_update(signing_digest, (uint8_t *)rbuf, sd->msglen);
943 /** Do compression if turned on */
944 if (ff_pkt->flags & FO_GZIP && jcr->pZLIB_compress_workset) {
945 Dmsg3(400, "cbuf=0x%x rbuf=0x%x len=%u\n", cbuf, rbuf, sd->msglen);
947 ((z_stream*)jcr->pZLIB_compress_workset)->next_in = (Bytef *)rbuf;
948 ((z_stream*)jcr->pZLIB_compress_workset)->avail_in = sd->msglen;
949 ((z_stream*)jcr->pZLIB_compress_workset)->next_out = (Bytef *)cbuf;
950 ((z_stream*)jcr->pZLIB_compress_workset)->avail_out = max_compress_len;
952 if ((zstat=deflate((z_stream*)jcr->pZLIB_compress_workset, Z_FINISH)) != Z_STREAM_END) {
953 Jmsg(jcr, M_FATAL, 0, _("Compression deflate error: %d\n"), zstat);
954 set_jcr_job_status(jcr, JS_ErrorTerminated);
957 compress_len = ((z_stream*)jcr->pZLIB_compress_workset)->total_out;
958 /** reset zlib stream to be able to begin from scratch again */
959 if ((zstat=deflateReset((z_stream*)jcr->pZLIB_compress_workset)) != Z_OK) {
960 Jmsg(jcr, M_FATAL, 0, _("Compression deflateReset error: %d\n"), zstat);
961 set_jcr_job_status(jcr, JS_ErrorTerminated);
965 Dmsg2(400, "compressed len=%d uncompressed len=%d\n", compress_len,
968 sd->msglen = compress_len; /* set compressed length */
969 cipher_input_len = compress_len;
973 * Note, here we prepend the current record length to the beginning
974 * of the encrypted data. This is because both sparse and compression
975 * restore handling want records returned to them with exactly the
976 * same number of bytes that were processed in the backup handling.
977 * That is, both are block filters rather than a stream. When doing
978 * compression, the compression routines may buffer data, so that for
979 * any one record compressed, when it is decompressed the same size
980 * will not be obtained. Of course, the buffered data eventually comes
981 * out in subsequent crypto_cipher_update() calls or at least
982 * when crypto_cipher_finalize() is called. Unfortunately, this
983 * "feature" of encryption enormously complicates the restore code.
985 if (ff_pkt->flags & FO_ENCRYPT) {
986 uint32_t initial_len = 0;
989 if (ff_pkt->flags & FO_SPARSE) {
990 cipher_input_len += SPARSE_FADDR_SIZE;
993 /** Encrypt the length of the input block */
994 uint8_t packet_len[sizeof(uint32_t)];
996 ser_begin(packet_len, sizeof(uint32_t));
997 ser_uint32(cipher_input_len); /* store data len in begin of buffer */
998 Dmsg1(20, "Encrypt len=%d\n", cipher_input_len);
1000 if (!crypto_cipher_update(cipher_ctx, packet_len, sizeof(packet_len),
1001 (uint8_t *)jcr->crypto.crypto_buf, &initial_len)) {
1002 /** Encryption failed. Shouldn't happen. */
1003 Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
1007 /** Encrypt the input block */
1008 if (crypto_cipher_update(cipher_ctx, cipher_input, cipher_input_len,
1009 (uint8_t *)&jcr->crypto.crypto_buf[initial_len], &encrypted_len)) {
1010 if ((initial_len + encrypted_len) == 0) {
1011 /** No full block of data available, read more data */
1014 Dmsg2(400, "encrypted len=%d unencrypted len=%d\n", encrypted_len,
1016 sd->msglen = initial_len + encrypted_len; /* set encrypted length */
1018 /** Encryption failed. Shouldn't happen. */
1019 Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
1024 /* Send the buffer to the Storage daemon */
1025 if (ff_pkt->flags & FO_SPARSE) {
1026 sd->msglen += SPARSE_FADDR_SIZE; /* include fileAddr in size */
1028 sd->msg = wbuf; /* set correct write buffer */
1030 if (!jcr->is_job_canceled()) {
1031 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1036 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
1038 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
1039 sd->msg = msgsave; /* restore read buffer */
1041 } /* end while read file data */
1043 if (sd->msglen < 0) { /* error */
1045 Jmsg(jcr, M_ERROR, 0, _("Read error on file %s. ERR=%s\n"),
1046 ff_pkt->fname, be.bstrerror(ff_pkt->bfd.berrno));
1047 if (jcr->JobErrors++ > 1000) { /* insanity check */
1048 Jmsg(jcr, M_FATAL, 0, _("Too many errors. JobErrors=%d.\n"), jcr->JobErrors);
1050 } else if (ff_pkt->flags & FO_ENCRYPT) {
1052 * For encryption, we must call finalize to push out any
1055 if (!crypto_cipher_finalize(cipher_ctx, (uint8_t *)jcr->crypto.crypto_buf,
1057 /* Padding failed. Shouldn't happen. */
1058 Jmsg(jcr, M_FATAL, 0, _("Encryption padding error\n"));
1062 /** Note, on SSL pre-0.9.7, there is always some output */
1063 if (encrypted_len > 0) {
1064 sd->msglen = encrypted_len; /* set encrypted length */
1065 sd->msg = jcr->crypto.crypto_buf; /* set correct write buffer */
1067 if (!jcr->is_job_canceled()) {
1068 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1073 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
1074 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
1075 sd->msg = msgsave; /* restore bnet buffer */
1079 if (!sd->signal(BNET_EOD)) { /* indicate end of file data */
1080 if (!jcr->is_job_canceled()) {
1081 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1087 /** Free the cipher context */
1089 crypto_cipher_free(cipher_ctx);
1094 /** Free the cipher context */
1096 crypto_cipher_free(cipher_ctx);
1099 sd->msg = msgsave; /* restore bnet buffer */
1104 bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream)
1106 BSOCK *sd = jcr->store_bsock;
1107 char attribs[MAXSTRING];
1108 char attribsExBuf[MAXSTRING];
1109 char *attribsEx = NULL;
1113 #ifdef FD_NO_SEND_TEST
1117 Dmsg1(300, "encode_and_send_attrs fname=%s\n", ff_pkt->fname);
1118 /** Find what data stream we will use, then encode the attributes */
1119 if ((data_stream = select_data_stream(ff_pkt)) == STREAM_NONE) {
1120 /* This should not happen */
1121 Jmsg0(jcr, M_FATAL, 0, _("Invalid file flags, no supported data stream type.\n"));
1124 encode_stat(attribs, &ff_pkt->statp, ff_pkt->LinkFI, data_stream);
1126 /** Now possibly extend the attributes */
1127 if (ff_pkt->type == FT_RESTORE_FIRST) {
1128 attr_stream = STREAM_RESTORE_OBJECT;
1130 attribsEx = attribsExBuf;
1131 attr_stream = encode_attribsEx(jcr, attribsEx, ff_pkt);
1134 Dmsg3(300, "File %s\nattribs=%s\nattribsEx=%s\n", ff_pkt->fname, attribs, attribsEx);
1137 jcr->JobFiles++; /* increment number of files sent */
1138 ff_pkt->FileIndex = jcr->JobFiles; /* return FileIndex */
1139 pm_strcpy(jcr->last_fname, ff_pkt->fname);
1143 * Send Attributes header to Storage daemon
1144 * <file-index> <stream> <info>
1146 if (!sd->fsend("%ld %d 0", jcr->JobFiles, attr_stream)) {
1147 if (!jcr->is_job_canceled()) {
1148 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1153 Dmsg1(300, ">stored: attrhdr %s\n", sd->msg);
1156 * Send file attributes to Storage daemon
1159 * Filename (full path)
1160 * Encoded attributes
1161 * Link name (if type==FT_LNK or FT_LNKSAVED)
1162 * Encoded extended-attributes (for Win32)
1164 * or send Restore Object to Storage daemon
1168 * Object_len (possibly compressed)
1169 * Object_full_len (not compressed)
1170 * Object_compression
1173 * Binary Object data
1175 * For a directory, link is the same as fname, but with trailing
1176 * slash. For a linked file, link is the link.
1178 if (ff_pkt->type != FT_DELETED) { /* already stripped */
1181 switch (ff_pkt->type) {
1184 Dmsg2(300, "Link %s to %s\n", ff_pkt->fname, ff_pkt->link);
1185 stat = sd->fsend("%ld %d %s%c%s%c%s%c%s%c", jcr->JobFiles,
1186 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, ff_pkt->link, 0,
1191 /* Here link is the canonical filename (i.e. with trailing slash) */
1192 stat = sd->fsend("%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
1193 ff_pkt->type, ff_pkt->link, 0, attribs, 0, 0, attribsEx, 0);
1195 case FT_RESTORE_FIRST:
1196 comp_len = ff_pkt->object_len;
1197 ff_pkt->object_compression = 0;
1198 if (ff_pkt->object_len > 1000) {
1199 /* Big object, compress it */
1201 comp_len = ff_pkt->object_len + 1000;
1202 POOLMEM *comp_obj = get_memory(comp_len);
1203 stat = Zdeflate(ff_pkt->object, ff_pkt->object_len, comp_obj, comp_len);
1204 if (comp_len < ff_pkt->object_len) {
1205 ff_pkt->object = comp_obj;
1206 ff_pkt->object_compression = 1; /* zlib level 9 compression */
1208 /* Uncompressed object smaller, use it */
1209 comp_len = ff_pkt->object_len;
1211 Dmsg2(100, "Object compressed from %d to %d bytes\n", ff_pkt->object_len, comp_len);
1213 sd->msglen = Mmsg(sd->msg, "%d %d %d %d %d %d %s%c%s%c",
1214 jcr->JobFiles, ff_pkt->type, ff_pkt->object_index,
1215 comp_len, ff_pkt->object_len, ff_pkt->object_compression,
1216 ff_pkt->fname, 0, ff_pkt->object_name, 0);
1217 sd->msg = check_pool_memory_size(sd->msg, sd->msglen + comp_len + 2);
1218 memcpy(sd->msg + sd->msglen, ff_pkt->object, comp_len);
1219 /* Note we send one extra byte so Dir can store zero after object */
1220 sd->msglen += comp_len + 1;
1222 if (ff_pkt->object_compression) {
1223 free_and_null_pool_memory(ff_pkt->object);
1227 stat = sd->fsend("%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
1228 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, 0, attribsEx, 0);
1231 if (ff_pkt->type != FT_DELETED) {
1232 unstrip_path(ff_pkt);
1235 Dmsg2(300, ">stored: attr len=%d: %s\n", sd->msglen, sd->msg);
1236 if (!stat && !jcr->is_job_canceled()) {
1237 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1240 sd->signal(BNET_EOD); /* indicate end of attributes data */
1245 * Do in place strip of path
1247 static bool do_strip(int count, char *in)
1253 /** Copy to first path separator -- Win32 might have c: ... */
1254 while (*in && !IsPathSeparator(*in)) {
1258 numsep++; /* one separator seen */
1259 for (stripped=0; stripped<count && *in; stripped++) {
1260 while (*in && !IsPathSeparator(*in)) {
1261 in++; /* skip chars */
1264 numsep++; /* count separators seen */
1265 in++; /* skip separator */
1269 while (*in) { /* copy to end */
1270 if (IsPathSeparator(*in)) {
1276 Dmsg4(500, "stripped=%d count=%d numsep=%d sep>count=%d\n",
1277 stripped, count, numsep, numsep>count);
1278 return stripped==count && numsep>count;
1282 * If requested strip leading components of the path so that we can
1283 * save file as if it came from a subdirectory. This is most useful
1284 * for dealing with snapshots, by removing the snapshot directory, or
1285 * in handling vendor migrations where files have been restored with
1286 * a vendor product into a subdirectory.
1288 void strip_path(FF_PKT *ff_pkt)
1290 if (!(ff_pkt->flags & FO_STRIPPATH) || ff_pkt->strip_path <= 0) {
1291 Dmsg1(200, "No strip for %s\n", ff_pkt->fname);
1294 if (!ff_pkt->fname_save) {
1295 ff_pkt->fname_save = get_pool_memory(PM_FNAME);
1296 ff_pkt->link_save = get_pool_memory(PM_FNAME);
1298 pm_strcpy(ff_pkt->fname_save, ff_pkt->fname);
1299 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1300 pm_strcpy(ff_pkt->link_save, ff_pkt->link);
1301 Dmsg2(500, "strcpy link_save=%d link=%d\n", strlen(ff_pkt->link_save),
1302 strlen(ff_pkt->link));
1303 sm_check(__FILE__, __LINE__, true);
1307 * Strip path. If it doesn't succeed put it back. If
1308 * it does, and there is a different link string,
1309 * attempt to strip the link. If it fails, back them
1311 * Do not strip symlinks.
1312 * I.e. if either stripping fails don't strip anything.
1314 if (!do_strip(ff_pkt->strip_path, ff_pkt->fname)) {
1315 unstrip_path(ff_pkt);
1318 /** Strip links but not symlinks */
1319 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1320 if (!do_strip(ff_pkt->strip_path, ff_pkt->link)) {
1321 unstrip_path(ff_pkt);
1326 Dmsg3(100, "fname=%s stripped=%s link=%s\n", ff_pkt->fname_save, ff_pkt->fname,
1330 void unstrip_path(FF_PKT *ff_pkt)
1332 if (!(ff_pkt->flags & FO_STRIPPATH) || ff_pkt->strip_path <= 0) {
1335 strcpy(ff_pkt->fname, ff_pkt->fname_save);
1336 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1337 Dmsg2(500, "strcpy link=%s link_save=%s\n", ff_pkt->link,
1339 strcpy(ff_pkt->link, ff_pkt->link_save);
1340 Dmsg2(500, "strcpy link=%d link_save=%d\n", strlen(ff_pkt->link),
1341 strlen(ff_pkt->link_save));
1342 sm_check(__FILE__, __LINE__, true);
1346 static void close_vss_backup_session(JCR *jcr)
1348 #if defined(WIN32_VSS)
1349 /* STOP VSS ON WIN32 */
1350 /* tell vss to close the backup session */
1352 if (g_pVSSClient->CloseBackup()) {
1353 /* inform user about writer states */
1354 for (int i=0; i<(int)g_pVSSClient->GetWriterCount(); i++) {
1355 int msg_type = M_INFO;
1356 if (g_pVSSClient->GetWriterState(i) < 1) {
1357 msg_type = M_WARNING;
1360 Jmsg(jcr, msg_type, 0, _("VSS Writer (BackupComplete): %s\n"), g_pVSSClient->GetWriterInfo(i));
1363 WCHAR *metadata = g_pVSSClient->GetMetadata();
1365 FF_PKT *ff_pkt = jcr->ff;
1366 ff_pkt->fname = "job";
1367 ff_pkt->type = FT_RESTORE_FIRST;
1369 ff_pkt->object_name = "job_metadata.xml";
1370 ff_pkt->object = (char *)metadata;
1371 ff_pkt->object_len = (wcslen(metadata) + 1) * sizeof(WCHAR);
1372 save_file(jcr, ff_pkt, true);