2 Bacula® - The Network Backup Solution
4 Copyright (C) 2000-2008 Free Software Foundation Europe e.V.
6 The main author of Bacula is Kern Sibbald, with contributions from
7 many others, a complete list can be found in the file AUTHORS.
8 This program is Free Software; you can redistribute it and/or
9 modify it under the terms of version two of the GNU General Public
10 License as published by the Free Software Foundation and included
13 This program is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
23 Bacula® is a registered trademark of Kern Sibbald.
24 The licensor of Bacula is the Free Software Foundation Europe
25 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
26 Switzerland, email:ftf@fsfeurope.org.
29 * Bacula File Daemon backup.c send file attributes and data
30 * to the Storage daemon.
32 * Kern Sibbald, March MM
41 /* Forward referenced functions */
42 int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level);
43 static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, DIGEST *signature_digest);
44 bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream);
45 static bool read_and_send_acl(JCR *jcr, int acltype, int stream);
46 static bool crypto_session_start(JCR *jcr);
47 static void crypto_session_end(JCR *jcr);
48 static bool crypto_session_send(JCR *jcr, BSOCK *sd);
51 * check for BSD nodump flag
53 static bool no_dump(JCR *jcr, FF_PKT *ff_pkt)
55 #if defined(HAVE_CHFLAGS) && defined(UF_NODUMP)
56 if ( (ff_pkt->flags & FO_HONOR_NODUMP) &&
57 (ff_pkt->statp.st_flags & UF_NODUMP) ) {
58 Jmsg(jcr, M_INFO, 1, _(" NODUMP flag set - will not process %s\n"),
60 return true; /* do not backup this file */
63 return false; /* do backup */
67 * Find all the requested files and send them
68 * to the Storage daemon.
70 * Note, we normally carry on a one-way
71 * conversation from this point on with the SD, simply blasting
72 * data to him. To properly know what is going on, we
73 * also run a "heartbeat" monitor which reads the socket and
74 * reacts accordingly (at the moment it has nothing to do
75 * except echo the heartbeat to the Director).
78 bool blast_data_to_storage_daemon(JCR *jcr, char *addr)
82 // TODO landonf: Allow user to specify encryption algorithm
84 sd = jcr->store_bsock;
86 set_jcr_job_status(jcr, JS_Running);
88 Dmsg1(300, "bfiled: opened data connection %d to stored\n", sd->m_fd);
91 CLIENT *client = (CLIENT *)GetNextRes(R_CLIENT, NULL);
95 buf_size = client->max_network_buffer_size;
97 buf_size = 0; /* use default */
99 if (!sd->set_buffer_size(buf_size, BNET_SETBUF_WRITE)) {
100 set_jcr_job_status(jcr, JS_ErrorTerminated);
101 Jmsg(jcr, M_FATAL, 0, _("Cannot set buffer size FD->SD.\n"));
105 jcr->buf_size = sd->msglen;
106 /* Adjust for compression so that output buffer is
107 * 12 bytes + 0.1% larger than input buffer plus 18 bytes.
108 * This gives a bit extra plus room for the sparse addr if any.
109 * Note, we adjust the read size to be smaller so that the
110 * same output buffer can be used without growing it.
112 * The zlib compression workset is initialized here to minimize
113 * the "per file" load. The jcr member is only set, if the init
116 jcr->compress_buf_size = jcr->buf_size + ((jcr->buf_size+999) / 1000) + 30;
117 jcr->compress_buf = get_memory(jcr->compress_buf_size);
120 z_stream *pZlibStream = (z_stream*)malloc(sizeof(z_stream));
122 pZlibStream->zalloc = Z_NULL;
123 pZlibStream->zfree = Z_NULL;
124 pZlibStream->opaque = Z_NULL;
125 pZlibStream->state = Z_NULL;
127 if (deflateInit(pZlibStream, Z_DEFAULT_COMPRESSION) == Z_OK) {
128 jcr->pZLIB_compress_workset = pZlibStream;
135 if (!crypto_session_start(jcr)) {
139 set_find_options((FF_PKT *)jcr->ff, jcr->incremental, jcr->mtime);
141 /* in accurate mode, we overwrite the find_one check function */
143 set_find_changed_function((FF_PKT *)jcr->ff, accurate_check_file);
146 start_heartbeat_monitor(jcr);
148 jcr->acl_text = get_pool_memory(PM_MESSAGE);
150 /* Subroutine save_file() is called for each file */
151 if (!find_files(jcr, (FF_PKT *)jcr->ff, save_file, plugin_save)) {
152 ok = false; /* error */
153 set_jcr_job_status(jcr, JS_ErrorTerminated);
156 accurate_send_deleted_list(jcr); /* send deleted list to SD */
158 free_pool_memory(jcr->acl_text);
160 stop_heartbeat_monitor(jcr);
162 sd->signal(BNET_EOD); /* end of sending data */
168 if (jcr->compress_buf) {
169 free_pool_memory(jcr->compress_buf);
170 jcr->compress_buf = NULL;
172 if (jcr->pZLIB_compress_workset) {
173 /* Free the zlib stream */
175 deflateEnd((z_stream *)jcr->pZLIB_compress_workset);
177 free (jcr->pZLIB_compress_workset);
178 jcr->pZLIB_compress_workset = NULL;
180 crypto_session_end(jcr);
183 Dmsg1(100, "end blast_data ok=%d\n", ok);
187 static bool crypto_session_start(JCR *jcr)
189 crypto_cipher_t cipher = CRYPTO_CIPHER_AES_128_CBC;
192 * Create encryption session data and a cached, DER-encoded session data
193 * structure. We use a single session key for each backup, so we'll encode
194 * the session data only once.
196 if (jcr->crypto.pki_encrypt) {
199 /* Create per-job session encryption context */
200 jcr->crypto.pki_session = crypto_session_new(cipher, jcr->crypto.pki_recipients);
202 /* Get the session data size */
203 if (!crypto_session_encode(jcr->crypto.pki_session, (uint8_t *)0, &size)) {
204 Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n"));
208 /* Allocate buffer */
209 jcr->crypto.pki_session_encoded = get_memory(size);
211 /* Encode session data */
212 if (!crypto_session_encode(jcr->crypto.pki_session, (uint8_t *)jcr->crypto.pki_session_encoded, &size)) {
213 Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n"));
217 /* ... and store the encoded size */
218 jcr->crypto.pki_session_encoded_size = size;
220 /* Allocate the encryption/decryption buffer */
221 jcr->crypto.crypto_buf = get_memory(CRYPTO_CIPHER_MAX_BLOCK_SIZE);
226 static void crypto_session_end(JCR *jcr)
228 if (jcr->crypto.crypto_buf) {
229 free_pool_memory(jcr->crypto.crypto_buf);
230 jcr->crypto.crypto_buf = NULL;
232 if (jcr->crypto.pki_session) {
233 crypto_session_free(jcr->crypto.pki_session);
235 if (jcr->crypto.pki_session_encoded) {
236 free_pool_memory(jcr->crypto.pki_session_encoded);
237 jcr->crypto.pki_session_encoded = NULL;
241 static bool crypto_session_send(JCR *jcr, BSOCK *sd)
245 /* Send our header */
246 Dmsg2(100, "Send hdr fi=%ld stream=%d\n", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA);
247 sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA);
250 sd->msg = jcr->crypto.pki_session_encoded;
251 sd->msglen = jcr->crypto.pki_session_encoded_size;
252 jcr->JobBytes += sd->msglen;
254 Dmsg1(100, "Send data len=%d\n", sd->msglen);
257 sd->signal(BNET_EOD);
263 * Called here by find() for each file included.
264 * This is a callback. The original is find_files() above.
266 * Send the file and its data to the Storage daemon.
270 * -1 to ignore file/directory (not used here)
272 int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level)
274 bool do_read = false;
275 int stat, data_stream;
277 DIGEST *digest = NULL;
278 DIGEST *signing_digest = NULL;
279 int digest_stream = STREAM_NONE;
280 SIGNATURE *sig = NULL;
281 bool has_file_data = false;
282 // TODO landonf: Allow the user to specify the digest algorithm
284 crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA256;
286 crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA1;
288 BSOCK *sd = jcr->store_bsock;
290 if (job_canceled(jcr)) {
294 jcr->num_files_examined++; /* bump total file count */
296 switch (ff_pkt->type) {
297 case FT_LNKSAVED: /* Hard linked, file already saved */
298 Dmsg2(130, "FT_LNKSAVED hard link: %s => %s\n", ff_pkt->fname, ff_pkt->link);
301 Dmsg1(130, "FT_REGE saving: %s\n", ff_pkt->fname);
302 if (no_dump(jcr, ff_pkt))
304 has_file_data = true;
307 Dmsg1(130, "FT_REG saving: %s\n", ff_pkt->fname);
308 if (no_dump(jcr, ff_pkt))
310 has_file_data = true;
313 Dmsg2(130, "FT_LNK saving: %s -> %s\n", ff_pkt->fname, ff_pkt->link);
316 jcr->num_files_examined--; /* correct file count */
317 if (no_dump(jcr, ff_pkt)) /* disable recursion on nodump directories */
318 ff_pkt->flags |= FO_NO_RECURSION;
319 return 1; /* not used */
321 Jmsg(jcr, M_INFO, 1, _(" Recursion turned off. Will not descend from %s into %s\n"),
322 ff_pkt->top_fname, ff_pkt->fname);
323 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
326 /* Suppress message for /dev filesystems */
327 if (!is_in_fileset(ff_pkt)) {
328 Jmsg(jcr, M_INFO, 1, _(" %s is a different filesystem. Will not descend from %s into %s\n"),
329 ff_pkt->fname, ff_pkt->top_fname, ff_pkt->fname);
331 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
334 Jmsg(jcr, M_INFO, 1, _(" Disallowed filesystem. Will not descend from %s into %s\n"),
335 ff_pkt->top_fname, ff_pkt->fname);
336 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
339 Jmsg(jcr, M_INFO, 1, _(" Disallowed drive type. Will not descend into %s\n"),
344 Dmsg1(130, "FT_DIREND: %s\n", ff_pkt->link);
347 Dmsg1(130, "FT_SPEC saving: %s\n", ff_pkt->fname);
348 if (S_ISSOCK(ff_pkt->statp.st_mode)) {
349 Jmsg(jcr, M_SKIPPED, 1, _(" Socket file skipped: %s\n"), ff_pkt->fname);
354 Dmsg1(130, "FT_RAW saving: %s\n", ff_pkt->fname);
355 has_file_data = true;
358 Dmsg1(130, "FT_FIFO saving: %s\n", ff_pkt->fname);
362 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not access \"%s\": ERR=%s\n"), ff_pkt->fname,
363 be.bstrerror(ff_pkt->ff_errno));
369 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not follow link \"%s\": ERR=%s\n"),
370 ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
376 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not stat \"%s\": ERR=%s\n"), ff_pkt->fname,
377 be.bstrerror(ff_pkt->ff_errno));
383 Jmsg(jcr, M_SKIPPED, 1, _(" Unchanged file skipped: %s\n"), ff_pkt->fname);
386 Jmsg(jcr, M_NOTSAVED, 0, _(" Archive file not saved: %s\n"), ff_pkt->fname);
390 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not open directory \"%s\": ERR=%s\n"),
391 ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
396 Jmsg(jcr, M_NOTSAVED, 0, _(" Unknown file type %d; not saved: %s\n"),
397 ff_pkt->type, ff_pkt->fname);
402 Dmsg1(130, "bfiled: sending %s to stored\n", ff_pkt->fname);
404 /* Digests and encryption are only useful if there's file data */
407 * Setup for digest handling. If this fails, the digest will be set to NULL
408 * and not used. Note, the digest (file hash) can be any one of the four
411 * The signing digest is a single algorithm depending on
412 * whether or not we have SHA2.
413 * ****FIXME**** the signing algoritm should really be
414 * determined a different way!!!!!! What happens if
415 * sha2 was available during backup but not restore?
417 if (ff_pkt->flags & FO_MD5) {
418 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_MD5);
419 digest_stream = STREAM_MD5_DIGEST;
421 } else if (ff_pkt->flags & FO_SHA1) {
422 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA1);
423 digest_stream = STREAM_SHA1_DIGEST;
425 } else if (ff_pkt->flags & FO_SHA256) {
426 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA256);
427 digest_stream = STREAM_SHA256_DIGEST;
429 } else if (ff_pkt->flags & FO_SHA512) {
430 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA512);
431 digest_stream = STREAM_SHA512_DIGEST;
434 /* Did digest initialization fail? */
435 if (digest_stream != STREAM_NONE && digest == NULL) {
436 Jmsg(jcr, M_WARNING, 0, _("%s digest initialization failed\n"),
437 stream_to_ascii(digest_stream));
441 * Set up signature digest handling. If this fails, the signature digest will be set to
444 // TODO landonf: We should really only calculate the digest once, for both verification and signing.
445 if (jcr->crypto.pki_sign) {
446 signing_digest = crypto_digest_new(jcr, signing_algorithm);
448 /* Full-stop if a failure occurred initializing the signature digest */
449 if (signing_digest == NULL) {
450 Jmsg(jcr, M_NOTSAVED, 0, _("%s signature digest initialization failed\n"),
451 stream_to_ascii(signing_algorithm));
457 /* Enable encryption */
458 if (jcr->crypto.pki_encrypt) {
459 ff_pkt->flags |= FO_ENCRYPT;
463 /* Initialize the file descriptor we use for data and other streams. */
465 if (ff_pkt->flags & FO_PORTABLE) {
466 set_portable_backup(&ff_pkt->bfd); /* disable Win32 BackupRead() */
468 if (ff_pkt->cmd_plugin) {
469 if (!set_cmd_plugin(&ff_pkt->bfd, jcr)) {
472 send_plugin_name(jcr, sd, true); /* signal start of plugin data */
475 /* Send attributes -- must be done after binit() */
476 if (!encode_and_send_attributes(jcr, ff_pkt, data_stream)) {
480 /* Set up the encryption context and send the session data to the SD */
481 if (has_file_data && jcr->crypto.pki_encrypt) {
482 if (!crypto_session_send(jcr, sd)) {
488 * Open any file with data that we intend to save, then save it.
490 * Note, if is_win32_backup, we must open the Directory so that
491 * the BackupRead will save its permissions and ownership streams.
493 if (ff_pkt->type != FT_LNKSAVED && S_ISREG(ff_pkt->statp.st_mode)) {
495 do_read = !is_portable_backup(&ff_pkt->bfd) || ff_pkt->statp.st_size > 0;
497 do_read = ff_pkt->statp.st_size > 0;
499 } else if (ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO ||
500 ff_pkt->type == FT_REPARSE ||
501 (!is_portable_backup(&ff_pkt->bfd) && ff_pkt->type == FT_DIREND)) {
504 if (ff_pkt->cmd_plugin) {
508 Dmsg1(100, "do_read=%d\n", do_read);
512 if (ff_pkt->type == FT_FIFO) {
513 tid = start_thread_timer(jcr, pthread_self(), 60);
517 int noatime = ff_pkt->flags & FO_NOATIME ? O_NOATIME : 0;
518 ff_pkt->bfd.reparse_point = ff_pkt->type == FT_REPARSE;
519 if (bopen(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY | noatime, 0) < 0) {
520 ff_pkt->ff_errno = errno;
522 Jmsg(jcr, M_NOTSAVED, 0, _(" Cannot open \"%s\": ERR=%s.\n"), ff_pkt->fname,
526 stop_thread_timer(tid);
532 stop_thread_timer(tid);
536 stat = send_data(jcr, data_stream, ff_pkt, digest, signing_digest);
538 if (ff_pkt->flags & FO_CHKCHANGES) {
539 has_file_changed(jcr, ff_pkt);
542 bclose(&ff_pkt->bfd);
549 #ifdef HAVE_DARWIN_OS
550 /* Regular files can have resource forks and Finder Info */
551 if (ff_pkt->type != FT_LNKSAVED && (S_ISREG(ff_pkt->statp.st_mode) &&
552 ff_pkt->flags & FO_HFSPLUS)) {
553 if (ff_pkt->hfsinfo.rsrclength > 0) {
556 if (!bopen_rsrc(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0) < 0) {
557 ff_pkt->ff_errno = errno;
559 Jmsg(jcr, M_NOTSAVED, -1, _(" Cannot open resource fork for \"%s\": ERR=%s.\n"),
560 ff_pkt->fname, be.bstrerror());
562 if (is_bopen(&ff_pkt->bfd)) {
563 bclose(&ff_pkt->bfd);
567 flags = ff_pkt->flags;
568 ff_pkt->flags &= ~(FO_GZIP|FO_SPARSE);
569 if (flags & FO_ENCRYPT) {
570 rsrc_stream = STREAM_ENCRYPTED_MACOS_FORK_DATA;
572 rsrc_stream = STREAM_MACOS_FORK_DATA;
574 stat = send_data(jcr, rsrc_stream, ff_pkt, digest, signing_digest);
575 ff_pkt->flags = flags;
576 bclose(&ff_pkt->bfd);
582 Dmsg1(300, "Saving Finder Info for \"%s\"\n", ff_pkt->fname);
583 sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_HFSPLUS_ATTRIBUTES);
584 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
585 memcpy(sd->msg, ff_pkt->hfsinfo.fndrinfo, 32);
588 crypto_digest_update(digest, (uint8_t *)sd->msg, sd->msglen);
590 if (signing_digest) {
591 crypto_digest_update(signing_digest, (uint8_t *)sd->msg, sd->msglen);
594 sd->signal(BNET_EOD);
598 if (ff_pkt->flags & FO_ACL) {
599 /* Read access ACLs for files, dirs and links */
600 if (!read_and_send_acl(jcr, BACL_TYPE_ACCESS, STREAM_UNIX_ACCESS_ACL)) {
603 /* Directories can have default ACLs too */
604 if (ff_pkt->type == FT_DIREND && (BACL_CAP & BACL_CAP_DEFAULTS_DIR)) {
605 if (!read_and_send_acl(jcr, BACL_TYPE_DEFAULT, STREAM_UNIX_DEFAULT_ACL)) {
611 /* Terminate the signing digest and send it to the Storage daemon */
612 if (signing_digest) {
615 if ((sig = crypto_sign_new(jcr)) == NULL) {
616 Jmsg(jcr, M_FATAL, 0, _("Failed to allocate memory for crypto signature.\n"));
620 if (!crypto_sign_add_signer(sig, signing_digest, jcr->crypto.pki_keypair)) {
621 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
625 /* Get signature size */
626 if (!crypto_sign_encode(sig, NULL, &size)) {
627 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
631 /* Grow the bsock buffer to fit our message if necessary */
632 if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
633 sd->msg = realloc_pool_memory(sd->msg, size);
636 /* Send our header */
637 sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_SIGNED_DIGEST);
638 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
640 /* Encode signature data */
641 if (!crypto_sign_encode(sig, (uint8_t *)sd->msg, &size)) {
642 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
648 sd->signal(BNET_EOD); /* end of checksum */
651 /* Terminate any digest and send it to Storage daemon */
655 sd->fsend("%ld %d 0", jcr->JobFiles, digest_stream);
656 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
658 size = CRYPTO_DIGEST_MAX_SIZE;
660 /* Grow the bsock buffer to fit our message if necessary */
661 if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
662 sd->msg = realloc_pool_memory(sd->msg, size);
665 if (!crypto_digest_finalize(digest, (uint8_t *)sd->msg, &size)) {
666 Jmsg(jcr, M_FATAL, 0, _("An error occurred finalizing signing the stream.\n"));
672 sd->signal(BNET_EOD); /* end of checksum */
674 if (ff_pkt->cmd_plugin) {
675 send_plugin_name(jcr, sd, false); /* signal end of plugin data */
679 rtnstat = 1; /* good return */
683 crypto_digest_free(digest);
685 if (signing_digest) {
686 crypto_digest_free(signing_digest);
689 crypto_sign_free(sig);
695 * Send data read from an already open file descriptor.
697 * We return 1 on sucess and 0 on errors.
700 * We use ff_pkt->statp.st_size when FO_SPARSE to know when to stop
702 * Currently this is not a problem as the only other stream, resource forks,
703 * are not handled as sparse files.
705 int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest,
706 DIGEST *signing_digest)
708 BSOCK *sd = jcr->store_bsock;
709 uint64_t fileAddr = 0; /* file address */
711 int32_t rsize = jcr->buf_size; /* read buffer size */
713 CIPHER_CONTEXT *cipher_ctx = NULL; /* Quell bogus uninitialized warnings */
714 const uint8_t *cipher_input;
715 uint32_t cipher_input_len;
716 uint32_t cipher_block_size;
717 uint32_t encrypted_len;
718 #ifdef FD_NO_SEND_TEST
723 rbuf = sd->msg; /* read buffer */
724 wbuf = sd->msg; /* write buffer */
725 cipher_input = (uint8_t *)rbuf; /* encrypt uncompressed data */
727 Dmsg1(300, "Saving data, type=%d\n", ff_pkt->type);
730 uLong compress_len = 0;
731 uLong max_compress_len = 0;
732 const Bytef *cbuf = NULL;
735 if (ff_pkt->flags & FO_GZIP) {
736 if (ff_pkt->flags & FO_SPARSE) {
737 cbuf = (Bytef *)jcr->compress_buf + SPARSE_FADDR_SIZE;
738 max_compress_len = jcr->compress_buf_size - SPARSE_FADDR_SIZE;
740 cbuf = (Bytef *)jcr->compress_buf;
741 max_compress_len = jcr->compress_buf_size; /* set max length */
743 wbuf = jcr->compress_buf; /* compressed output here */
744 cipher_input = (uint8_t *)jcr->compress_buf; /* encrypt compressed data */
747 * Only change zlib parameters if there is no pending operation.
748 * This should never happen as deflatereset is called after each
752 if (((z_stream*)jcr->pZLIB_compress_workset)->total_in == 0) {
753 /* set gzip compression level - must be done per file */
754 if ((zstat=deflateParams((z_stream*)jcr->pZLIB_compress_workset,
755 ff_pkt->GZIP_level, Z_DEFAULT_STRATEGY)) != Z_OK) {
756 Jmsg(jcr, M_FATAL, 0, _("Compression deflateParams error: %d\n"), zstat);
757 set_jcr_job_status(jcr, JS_ErrorTerminated);
763 const uint32_t max_compress_len = 0;
766 if (ff_pkt->flags & FO_ENCRYPT) {
767 if (ff_pkt->flags & FO_SPARSE) {
768 Jmsg0(jcr, M_FATAL, 0, _("Encrypting sparse data not supported.\n"));
771 /* Allocate the cipher context */
772 if ((cipher_ctx = crypto_cipher_new(jcr->crypto.pki_session, true,
773 &cipher_block_size)) == NULL) {
774 /* Shouldn't happen! */
775 Jmsg0(jcr, M_FATAL, 0, _("Failed to initialize encryption context.\n"));
780 * Grow the crypto buffer, if necessary.
781 * crypto_cipher_update() will buffer up to (cipher_block_size - 1).
782 * We grow crypto_buf to the maximum number of blocks that
783 * could be returned for the given read buffer size.
784 * (Using the larger of either rsize or max_compress_len)
786 jcr->crypto.crypto_buf = check_pool_memory_size(jcr->crypto.crypto_buf,
787 (MAX(rsize + (int)sizeof(uint32_t), (int32_t)max_compress_len) +
788 cipher_block_size - 1) / cipher_block_size * cipher_block_size);
790 wbuf = jcr->crypto.crypto_buf; /* Encrypted, possibly compressed output here. */
794 * Send Data header to Storage daemon
795 * <file-index> <stream> <info>
797 if (!sd->fsend("%ld %d 0", jcr->JobFiles, stream)) {
798 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
802 Dmsg1(300, ">stored: datahdr %s\n", sd->msg);
805 * Make space at beginning of buffer for fileAddr because this
806 * same buffer will be used for writing if compression is off.
808 if (ff_pkt->flags & FO_SPARSE) {
809 rbuf += SPARSE_FADDR_SIZE;
810 rsize -= SPARSE_FADDR_SIZE;
811 #ifdef HAVE_FREEBSD_OS
813 * To read FreeBSD partitions, the read size must be
816 rsize = (rsize/512) * 512;
820 /* a RAW device read on win32 only works if the buffer is a multiple of 512 */
822 if (S_ISBLK(ff_pkt->statp.st_mode))
823 rsize = (rsize/512) * 512;
829 while ((sd->msglen=(uint32_t)bread(&ff_pkt->bfd, rbuf, rsize)) > 0) {
831 /* Check for sparse blocks */
832 if (ff_pkt->flags & FO_SPARSE) {
834 bool haveBlock = true;
835 if (sd->msglen == rsize &&
836 fileAddr+sd->msglen < (uint64_t)ff_pkt->statp.st_size ||
837 ((ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO) &&
838 (uint64_t)ff_pkt->statp.st_size == 0)) {
839 haveBlock = !is_buf_zero(rbuf, rsize);
842 ser_begin(wbuf, SPARSE_FADDR_SIZE);
843 ser_uint64(fileAddr); /* store fileAddr in begin of buffer */
845 fileAddr += sd->msglen; /* update file address */
847 continue; /* skip block of zeros */
851 jcr->ReadBytes += sd->msglen; /* count bytes read */
853 /* Uncompressed cipher input length */
854 cipher_input_len = sd->msglen;
856 /* Update checksum if requested */
858 crypto_digest_update(digest, (uint8_t *)rbuf, sd->msglen);
861 /* Update signing digest if requested */
862 if (signing_digest) {
863 crypto_digest_update(signing_digest, (uint8_t *)rbuf, sd->msglen);
867 /* Do compression if turned on */
868 if (ff_pkt->flags & FO_GZIP && jcr->pZLIB_compress_workset) {
869 Dmsg3(400, "cbuf=0x%x rbuf=0x%x len=%u\n", cbuf, rbuf, sd->msglen);
871 ((z_stream*)jcr->pZLIB_compress_workset)->next_in = (Bytef *)rbuf;
872 ((z_stream*)jcr->pZLIB_compress_workset)->avail_in = sd->msglen;
873 ((z_stream*)jcr->pZLIB_compress_workset)->next_out = (Bytef *)cbuf;
874 ((z_stream*)jcr->pZLIB_compress_workset)->avail_out = max_compress_len;
876 if ((zstat=deflate((z_stream*)jcr->pZLIB_compress_workset, Z_FINISH)) != Z_STREAM_END) {
877 Jmsg(jcr, M_FATAL, 0, _("Compression deflate error: %d\n"), zstat);
878 set_jcr_job_status(jcr, JS_ErrorTerminated);
881 compress_len = ((z_stream*)jcr->pZLIB_compress_workset)->total_out;
882 /* reset zlib stream to be able to begin from scratch again */
883 if ((zstat=deflateReset((z_stream*)jcr->pZLIB_compress_workset)) != Z_OK) {
884 Jmsg(jcr, M_FATAL, 0, _("Compression deflateReset error: %d\n"), zstat);
885 set_jcr_job_status(jcr, JS_ErrorTerminated);
889 Dmsg2(400, "compressed len=%d uncompressed len=%d\n", compress_len,
892 sd->msglen = compress_len; /* set compressed length */
893 cipher_input_len = compress_len;
897 * Note, here we prepend the current record length to the beginning
898 * of the encrypted data. This is because both sparse and compression
899 * restore handling want records returned to them with exactly the
900 * same number of bytes that were processed in the backup handling.
901 * That is, both are block filters rather than a stream. When doing
902 * compression, the compression routines may buffer data, so that for
903 * any one record compressed, when it is decompressed the same size
904 * will not be obtained. Of course, the buffered data eventually comes
905 * out in subsequent crypto_cipher_update() calls or at least
906 * when crypto_cipher_finalize() is called. Unfortunately, this
907 * "feature" of encryption enormously complicates the restore code.
909 if (ff_pkt->flags & FO_ENCRYPT) {
910 uint32_t initial_len = 0;
913 if (ff_pkt->flags & FO_SPARSE) {
914 cipher_input_len += SPARSE_FADDR_SIZE;
917 /* Encrypt the length of the input block */
918 uint8_t packet_len[sizeof(uint32_t)];
920 ser_begin(packet_len, sizeof(uint32_t));
921 ser_uint32(cipher_input_len); /* store data len in begin of buffer */
922 Dmsg1(20, "Encrypt len=%d\n", cipher_input_len);
924 if (!crypto_cipher_update(cipher_ctx, packet_len, sizeof(packet_len),
925 (uint8_t *)jcr->crypto.crypto_buf, &initial_len)) {
926 /* Encryption failed. Shouldn't happen. */
927 Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
931 /* Encrypt the input block */
932 if (crypto_cipher_update(cipher_ctx, cipher_input, cipher_input_len,
933 (uint8_t *)&jcr->crypto.crypto_buf[initial_len], &encrypted_len)) {
934 if ((initial_len + encrypted_len) == 0) {
935 /* No full block of data available, read more data */
938 Dmsg2(400, "encrypted len=%d unencrypted len=%d\n", encrypted_len,
940 sd->msglen = initial_len + encrypted_len; /* set encrypted length */
942 /* Encryption failed. Shouldn't happen. */
943 Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
948 /* Send the buffer to the Storage daemon */
949 if (ff_pkt->flags & FO_SPARSE) {
950 sd->msglen += SPARSE_FADDR_SIZE; /* include fileAddr in size */
952 sd->msg = wbuf; /* set correct write buffer */
954 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
958 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
960 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
961 sd->msg = msgsave; /* restore read buffer */
963 } /* end while read file data */
965 if (sd->msglen < 0) { /* error */
967 Jmsg(jcr, M_ERROR, 0, _("Read error on file %s. ERR=%s\n"),
968 ff_pkt->fname, be.bstrerror(ff_pkt->bfd.berrno));
969 if (jcr->Errors++ > 1000) { /* insanity check */
970 Jmsg(jcr, M_FATAL, 0, _("Too many errors.\n"));
972 } else if (ff_pkt->flags & FO_ENCRYPT) {
974 * For encryption, we must call finalize to push out any
977 if (!crypto_cipher_finalize(cipher_ctx, (uint8_t *)jcr->crypto.crypto_buf,
979 /* Padding failed. Shouldn't happen. */
980 Jmsg(jcr, M_FATAL, 0, _("Encryption padding error\n"));
984 /* Note, on SSL pre-0.9.7, there is always some output */
985 if (encrypted_len > 0) {
986 sd->msglen = encrypted_len; /* set encrypted length */
987 sd->msg = jcr->crypto.crypto_buf; /* set correct write buffer */
989 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
993 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
994 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
995 sd->msg = msgsave; /* restore bnet buffer */
999 if (!sd->signal(BNET_EOD)) { /* indicate end of file data */
1000 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1005 /* Free the cipher context */
1007 crypto_cipher_free(cipher_ctx);
1012 /* Free the cipher context */
1014 crypto_cipher_free(cipher_ctx);
1017 sd->msg = msgsave; /* restore bnet buffer */
1023 * Read and send an ACL for the last encountered file.
1025 static bool read_and_send_acl(JCR *jcr, int acltype, int stream)
1028 BSOCK *sd = jcr->store_bsock;
1031 #ifdef FD_NO_SEND_TEST
1035 len = bacl_get(jcr, acltype);
1037 Jmsg1(jcr, M_WARNING, 0, _("Error reading ACL of %s\n"), jcr->last_fname);
1041 return true; /* no ACL */
1045 if (!sd->fsend("%ld %d 0", jcr->JobFiles, stream)) {
1046 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1051 /* Send the buffer to the storage deamon */
1052 Dmsg2(400, "Backing up ACL type 0x%2x <%s>\n", acltype, jcr->acl_text);
1054 sd->msg = jcr->acl_text;
1055 sd->msglen = len + 1;
1059 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1064 jcr->JobBytes += sd->msglen;
1066 if (!sd->signal(BNET_EOD)) {
1067 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1072 Dmsg1(200, "ACL of file: %s successfully backed up!\n", jcr->last_fname);
1077 bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream)
1079 BSOCK *sd = jcr->store_bsock;
1080 char attribs[MAXSTRING];
1081 char attribsEx[MAXSTRING];
1084 #ifdef FD_NO_SEND_TEST
1088 Dmsg1(300, "encode_and_send_attrs fname=%s\n", ff_pkt->fname);
1089 /* Find what data stream we will use, then encode the attributes */
1090 if ((data_stream = select_data_stream(ff_pkt)) == STREAM_NONE) {
1091 /* This should not happen */
1092 Jmsg0(jcr, M_FATAL, 0, _("Invalid file flags, no supported data stream type.\n"));
1095 encode_stat(attribs, ff_pkt, data_stream);
1097 /* Now possibly extend the attributes */
1098 attr_stream = encode_attribsEx(jcr, attribsEx, ff_pkt);
1100 Dmsg3(300, "File %s\nattribs=%s\nattribsEx=%s\n", ff_pkt->fname, attribs, attribsEx);
1103 jcr->JobFiles++; /* increment number of files sent */
1104 ff_pkt->FileIndex = jcr->JobFiles; /* return FileIndex */
1105 pm_strcpy(jcr->last_fname, ff_pkt->fname);
1109 * Send Attributes header to Storage daemon
1110 * <file-index> <stream> <info>
1112 if (!sd->fsend("%ld %d 0", jcr->JobFiles, attr_stream)) {
1113 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1117 Dmsg1(300, ">stored: attrhdr %s\n", sd->msg);
1120 * Send file attributes to Storage daemon
1123 * Filename (full path)
1124 * Encoded attributes
1125 * Link name (if type==FT_LNK or FT_LNKSAVED)
1126 * Encoded extended-attributes (for Win32)
1128 * For a directory, link is the same as fname, but with trailing
1129 * slash. For a linked file, link is the link.
1131 if (ff_pkt->type != FT_DELETED) { /* already stripped */
1134 if (ff_pkt->type == FT_LNK || ff_pkt->type == FT_LNKSAVED) {
1135 Dmsg2(300, "Link %s to %s\n", ff_pkt->fname, ff_pkt->link);
1136 stat = sd->fsend("%ld %d %s%c%s%c%s%c%s%c", jcr->JobFiles,
1137 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, ff_pkt->link, 0,
1139 } else if (ff_pkt->type == FT_DIREND || ff_pkt->type == FT_REPARSE) {
1140 /* Here link is the canonical filename (i.e. with trailing slash) */
1141 stat = sd->fsend("%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
1142 ff_pkt->type, ff_pkt->link, 0, attribs, 0, 0, attribsEx, 0);
1144 stat = sd->fsend("%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
1145 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, 0, attribsEx, 0);
1147 if (ff_pkt->type != FT_DELETED) {
1148 unstrip_path(ff_pkt);
1151 Dmsg2(300, ">stored: attr len=%d: %s\n", sd->msglen, sd->msg);
1153 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1157 sd->signal(BNET_EOD); /* indicate end of attributes data */
1162 * Do in place strip of path
1164 static bool do_strip(int count, char *in)
1170 /* Copy to first path separator -- Win32 might have c: ... */
1171 while (*in && !IsPathSeparator(*in)) {
1175 numsep++; /* one separator seen */
1176 for (stripped=0; stripped<count && *in; stripped++) {
1177 while (*in && !IsPathSeparator(*in)) {
1178 in++; /* skip chars */
1181 numsep++; /* count separators seen */
1182 in++; /* skip separator */
1186 while (*in) { /* copy to end */
1187 if (IsPathSeparator(*in)) {
1193 Dmsg4(500, "stripped=%d count=%d numsep=%d sep>count=%d\n",
1194 stripped, count, numsep, numsep>count);
1195 return stripped==count && numsep>count;
1199 * If requested strip leading components of the path so that we can
1200 * save file as if it came from a subdirectory. This is most useful
1201 * for dealing with snapshots, by removing the snapshot directory, or
1202 * in handling vendor migrations where files have been restored with
1203 * a vendor product into a subdirectory.
1205 void strip_path(FF_PKT *ff_pkt)
1207 if (!(ff_pkt->flags & FO_STRIPPATH) || ff_pkt->strip_path <= 0) {
1208 Dmsg1(200, "No strip for %s\n", ff_pkt->fname);
1211 if (!ff_pkt->fname_save) {
1212 ff_pkt->fname_save = get_pool_memory(PM_FNAME);
1213 ff_pkt->link_save = get_pool_memory(PM_FNAME);
1215 pm_strcpy(ff_pkt->fname_save, ff_pkt->fname);
1216 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1217 pm_strcpy(ff_pkt->link_save, ff_pkt->link);
1218 Dmsg2(500, "strcpy link_save=%d link=%d\n", strlen(ff_pkt->link_save),
1219 strlen(ff_pkt->link));
1220 sm_check(__FILE__, __LINE__, true);
1224 * Strip path. If it doesn't succeed put it back. If
1225 * it does, and there is a different link string,
1226 * attempt to strip the link. If it fails, back them
1228 * Do not strip symlinks.
1229 * I.e. if either stripping fails don't strip anything.
1231 if (!do_strip(ff_pkt->strip_path, ff_pkt->fname)) {
1232 unstrip_path(ff_pkt);
1235 /* Strip links but not symlinks */
1236 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1237 if (!do_strip(ff_pkt->strip_path, ff_pkt->link)) {
1238 unstrip_path(ff_pkt);
1243 Dmsg3(100, "fname=%s stripped=%s link=%s\n", ff_pkt->fname_save, ff_pkt->fname,
1247 void unstrip_path(FF_PKT *ff_pkt)
1249 if (!(ff_pkt->flags & FO_STRIPPATH) || ff_pkt->strip_path <= 0) {
1252 strcpy(ff_pkt->fname, ff_pkt->fname_save);
1253 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1254 Dmsg2(500, "strcpy link=%s link_save=%s\n", ff_pkt->link,
1256 strcpy(ff_pkt->link, ff_pkt->link_save);
1257 Dmsg2(500, "strcpy link=%d link_save=%d\n", strlen(ff_pkt->link),
1258 strlen(ff_pkt->link_save));
1259 sm_check(__FILE__, __LINE__, true);