2 Bacula® - The Network Backup Solution
4 Copyright (C) 2000-2008 Free Software Foundation Europe e.V.
6 The main author of Bacula is Kern Sibbald, with contributions from
7 many others, a complete list can be found in the file AUTHORS.
8 This program is Free Software; you can redistribute it and/or
9 modify it under the terms of version two of the GNU General Public
10 License as published by the Free Software Foundation and included
13 This program is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
23 Bacula® is a registered trademark of John Walker.
24 The licensor of Bacula is the Free Software Foundation Europe
25 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
26 Switzerland, email:ftf@fsfeurope.org.
29 * Bacula File Daemon backup.c send file attributes and data
30 * to the Storage daemon.
32 * Kern Sibbald, March MM
42 bool accurate_send_deleted_list(JCR *jcr);
43 bool accurate_check_file(JCR *jcr, FF_PKT *ff_pkt);
45 /* Forward referenced functions */
46 int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level);
47 void strip_path(FF_PKT *ff_pkt);
48 void unstrip_path(FF_PKT *ff_pkt);
49 static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, DIGEST *signature_digest);
50 bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream);
51 static bool read_and_send_acl(JCR *jcr, int acltype, int stream);
52 static bool crypto_session_start(JCR *jcr);
53 static void crypto_session_end(JCR *jcr);
54 static bool crypto_session_send(JCR *jcr, BSOCK *sd);
57 * check for BSD nodump flag
59 static bool no_dump(JCR *jcr, FF_PKT *ff_pkt)
61 #if defined(HAVE_CHFLAGS) && defined(UF_NODUMP)
62 if ( (ff_pkt->flags & FO_HONOR_NODUMP) &&
63 (ff_pkt->statp.st_flags & UF_NODUMP) ) {
64 Jmsg(jcr, M_INFO, 1, _(" NODUMP flag set - will not process %s\n"),
66 return true; /* do not backup this file */
69 return false; /* do backup */
73 * Find all the requested files and send them
74 * to the Storage daemon.
76 * Note, we normally carry on a one-way
77 * conversation from this point on with the SD, simply blasting
78 * data to him. To properly know what is going on, we
79 * also run a "heartbeat" monitor which reads the socket and
80 * reacts accordingly (at the moment it has nothing to do
81 * except echo the heartbeat to the Director).
84 bool blast_data_to_storage_daemon(JCR *jcr, char *addr)
88 // TODO landonf: Allow user to specify encryption algorithm
90 sd = jcr->store_bsock;
92 set_jcr_job_status(jcr, JS_Running);
94 Dmsg1(300, "bfiled: opened data connection %d to stored\n", sd->m_fd);
97 CLIENT *client = (CLIENT *)GetNextRes(R_CLIENT, NULL);
101 buf_size = client->max_network_buffer_size;
103 buf_size = 0; /* use default */
105 if (!sd->set_buffer_size(buf_size, BNET_SETBUF_WRITE)) {
106 set_jcr_job_status(jcr, JS_ErrorTerminated);
107 Jmsg(jcr, M_FATAL, 0, _("Cannot set buffer size FD->SD.\n"));
111 jcr->buf_size = sd->msglen;
112 /* Adjust for compression so that output buffer is
113 * 12 bytes + 0.1% larger than input buffer plus 18 bytes.
114 * This gives a bit extra plus room for the sparse addr if any.
115 * Note, we adjust the read size to be smaller so that the
116 * same output buffer can be used without growing it.
118 * The zlib compression workset is initialized here to minimize
119 * the "per file" load. The jcr member is only set, if the init
122 jcr->compress_buf_size = jcr->buf_size + ((jcr->buf_size+999) / 1000) + 30;
123 jcr->compress_buf = get_memory(jcr->compress_buf_size);
126 z_stream *pZlibStream = (z_stream*)malloc(sizeof(z_stream));
128 pZlibStream->zalloc = Z_NULL;
129 pZlibStream->zfree = Z_NULL;
130 pZlibStream->opaque = Z_NULL;
131 pZlibStream->state = Z_NULL;
133 if (deflateInit(pZlibStream, Z_DEFAULT_COMPRESSION) == Z_OK) {
134 jcr->pZLIB_compress_workset = pZlibStream;
141 if (!crypto_session_start(jcr)) {
145 set_find_options((FF_PKT *)jcr->ff, jcr->incremental, jcr->mtime);
147 /* in accurate mode, we overwrite the find_one check function */
149 set_find_changed_function((FF_PKT *)jcr->ff, accurate_check_file);
152 start_heartbeat_monitor(jcr);
154 jcr->acl_text = get_pool_memory(PM_MESSAGE);
156 /* Subroutine save_file() is called for each file */
157 if (!find_files(jcr, (FF_PKT *)jcr->ff, save_file, plugin_save)) {
158 ok = false; /* error */
159 set_jcr_job_status(jcr, JS_ErrorTerminated);
162 accurate_send_deleted_list(jcr); /* send deleted list to SD */
164 free_pool_memory(jcr->acl_text);
166 stop_heartbeat_monitor(jcr);
168 sd->signal(BNET_EOD); /* end of sending data */
174 if (jcr->compress_buf) {
175 free_pool_memory(jcr->compress_buf);
176 jcr->compress_buf = NULL;
178 if (jcr->pZLIB_compress_workset) {
179 /* Free the zlib stream */
181 deflateEnd((z_stream *)jcr->pZLIB_compress_workset);
183 free (jcr->pZLIB_compress_workset);
184 jcr->pZLIB_compress_workset = NULL;
186 crypto_session_end(jcr);
189 Dmsg1(100, "end blast_data ok=%d\n", ok);
193 static bool crypto_session_start(JCR *jcr)
195 crypto_cipher_t cipher = CRYPTO_CIPHER_AES_128_CBC;
198 * Create encryption session data and a cached, DER-encoded session data
199 * structure. We use a single session key for each backup, so we'll encode
200 * the session data only once.
202 if (jcr->crypto.pki_encrypt) {
205 /* Create per-job session encryption context */
206 jcr->crypto.pki_session = crypto_session_new(cipher, jcr->crypto.pki_recipients);
208 /* Get the session data size */
209 if (!crypto_session_encode(jcr->crypto.pki_session, (uint8_t *)0, &size)) {
210 Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n"));
214 /* Allocate buffer */
215 jcr->crypto.pki_session_encoded = get_memory(size);
217 /* Encode session data */
218 if (!crypto_session_encode(jcr->crypto.pki_session, (uint8_t *)jcr->crypto.pki_session_encoded, &size)) {
219 Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n"));
223 /* ... and store the encoded size */
224 jcr->crypto.pki_session_encoded_size = size;
226 /* Allocate the encryption/decryption buffer */
227 jcr->crypto.crypto_buf = get_memory(CRYPTO_CIPHER_MAX_BLOCK_SIZE);
232 static void crypto_session_end(JCR *jcr)
234 if (jcr->crypto.crypto_buf) {
235 free_pool_memory(jcr->crypto.crypto_buf);
236 jcr->crypto.crypto_buf = NULL;
238 if (jcr->crypto.pki_session) {
239 crypto_session_free(jcr->crypto.pki_session);
241 if (jcr->crypto.pki_session_encoded) {
242 free_pool_memory(jcr->crypto.pki_session_encoded);
243 jcr->crypto.pki_session_encoded = NULL;
247 static bool crypto_session_send(JCR *jcr, BSOCK *sd)
251 /* Send our header */
252 Dmsg2(100, "Send hdr fi=%ld stream=%d\n", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA);
253 sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA);
256 sd->msg = jcr->crypto.pki_session_encoded;
257 sd->msglen = jcr->crypto.pki_session_encoded_size;
258 jcr->JobBytes += sd->msglen;
260 Dmsg1(100, "Send data len=%d\n", sd->msglen);
263 sd->signal(BNET_EOD);
269 * Called here by find() for each file included.
270 * This is a callback. The original is find_files() above.
272 * Send the file and its data to the Storage daemon.
276 * -1 to ignore file/directory (not used here)
278 int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level)
280 bool do_read = false;
281 int stat, data_stream;
283 DIGEST *digest = NULL;
284 DIGEST *signing_digest = NULL;
285 int digest_stream = STREAM_NONE;
286 SIGNATURE *sig = NULL;
287 bool has_file_data = false;
288 // TODO landonf: Allow the user to specify the digest algorithm
290 crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA256;
292 crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA1;
294 BSOCK *sd = jcr->store_bsock;
296 if (job_canceled(jcr)) {
300 jcr->num_files_examined++; /* bump total file count */
302 switch (ff_pkt->type) {
303 case FT_LNKSAVED: /* Hard linked, file already saved */
304 Dmsg2(130, "FT_LNKSAVED hard link: %s => %s\n", ff_pkt->fname, ff_pkt->link);
307 Dmsg1(130, "FT_REGE saving: %s\n", ff_pkt->fname);
308 if (no_dump(jcr, ff_pkt))
310 has_file_data = true;
313 Dmsg1(130, "FT_REG saving: %s\n", ff_pkt->fname);
314 if (no_dump(jcr, ff_pkt))
316 has_file_data = true;
319 Dmsg2(130, "FT_LNK saving: %s -> %s\n", ff_pkt->fname, ff_pkt->link);
322 jcr->num_files_examined--; /* correct file count */
323 if (no_dump(jcr, ff_pkt)) /* disable recursion on nodump directories */
324 ff_pkt->flags |= FO_NO_RECURSION;
325 return 1; /* not used */
327 Jmsg(jcr, M_INFO, 1, _(" Recursion turned off. Will not descend from %s into %s\n"),
328 ff_pkt->top_fname, ff_pkt->fname);
329 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
332 /* Suppress message for /dev filesystems */
333 if (!is_in_fileset(ff_pkt)) {
334 Jmsg(jcr, M_INFO, 1, _(" %s is a different filesystem. Will not descend from %s into %s\n"),
335 ff_pkt->fname, ff_pkt->top_fname, ff_pkt->fname);
337 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
340 Jmsg(jcr, M_INFO, 1, _(" Disallowed filesystem. Will not descend from %s into %s\n"),
341 ff_pkt->top_fname, ff_pkt->fname);
342 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
345 Jmsg(jcr, M_INFO, 1, _(" Disallowed drive type. Will not descend into %s\n"),
350 Dmsg1(130, "FT_DIREND: %s\n", ff_pkt->link);
353 Dmsg1(130, "FT_SPEC saving: %s\n", ff_pkt->fname);
354 if (S_ISSOCK(ff_pkt->statp.st_mode)) {
355 Jmsg(jcr, M_SKIPPED, 1, _(" Socket file skipped: %s\n"), ff_pkt->fname);
360 Dmsg1(130, "FT_RAW saving: %s\n", ff_pkt->fname);
361 has_file_data = true;
364 Dmsg1(130, "FT_FIFO saving: %s\n", ff_pkt->fname);
368 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not access \"%s\": ERR=%s\n"), ff_pkt->fname,
369 be.bstrerror(ff_pkt->ff_errno));
375 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not follow link \"%s\": ERR=%s\n"),
376 ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
382 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not stat \"%s\": ERR=%s\n"), ff_pkt->fname,
383 be.bstrerror(ff_pkt->ff_errno));
389 Jmsg(jcr, M_SKIPPED, 1, _(" Unchanged file skipped: %s\n"), ff_pkt->fname);
392 Jmsg(jcr, M_NOTSAVED, 0, _(" Archive file not saved: %s\n"), ff_pkt->fname);
396 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not open directory \"%s\": ERR=%s\n"),
397 ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
402 Jmsg(jcr, M_NOTSAVED, 0, _(" Unknown file type %d; not saved: %s\n"),
403 ff_pkt->type, ff_pkt->fname);
408 Dmsg1(130, "bfiled: sending %s to stored\n", ff_pkt->fname);
410 /* Digests and encryption are only useful if there's file data */
413 * Setup for digest handling. If this fails, the digest will be set to NULL
414 * and not used. Note, the digest (file hash) can be any one of the four
417 * The signing digest is a single algorithm depending on
418 * whether or not we have SHA2.
419 * ****FIXME**** the signing algoritm should really be
420 * determined a different way!!!!!! What happens if
421 * sha2 was available during backup but not restore?
423 if (ff_pkt->flags & FO_MD5) {
424 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_MD5);
425 digest_stream = STREAM_MD5_DIGEST;
427 } else if (ff_pkt->flags & FO_SHA1) {
428 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA1);
429 digest_stream = STREAM_SHA1_DIGEST;
431 } else if (ff_pkt->flags & FO_SHA256) {
432 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA256);
433 digest_stream = STREAM_SHA256_DIGEST;
435 } else if (ff_pkt->flags & FO_SHA512) {
436 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA512);
437 digest_stream = STREAM_SHA512_DIGEST;
440 /* Did digest initialization fail? */
441 if (digest_stream != STREAM_NONE && digest == NULL) {
442 Jmsg(jcr, M_WARNING, 0, _("%s digest initialization failed\n"),
443 stream_to_ascii(digest_stream));
447 * Set up signature digest handling. If this fails, the signature digest will be set to
450 // TODO landonf: We should really only calculate the digest once, for both verification and signing.
451 if (jcr->crypto.pki_sign) {
452 signing_digest = crypto_digest_new(jcr, signing_algorithm);
454 /* Full-stop if a failure occurred initializing the signature digest */
455 if (signing_digest == NULL) {
456 Jmsg(jcr, M_NOTSAVED, 0, _("%s signature digest initialization failed\n"),
457 stream_to_ascii(signing_algorithm));
463 /* Enable encryption */
464 if (jcr->crypto.pki_encrypt) {
465 ff_pkt->flags |= FO_ENCRYPT;
469 /* Initialize the file descriptor we use for data and other streams. */
471 if (ff_pkt->flags & FO_PORTABLE) {
472 set_portable_backup(&ff_pkt->bfd); /* disable Win32 BackupRead() */
474 if (ff_pkt->cmd_plugin) {
475 if (!set_cmd_plugin(&ff_pkt->bfd, jcr)) {
478 send_plugin_name(jcr, sd, true); /* signal start of plugin data */
481 /* Send attributes -- must be done after binit() */
482 if (!encode_and_send_attributes(jcr, ff_pkt, data_stream)) {
486 /* Set up the encryption context and send the session data to the SD */
487 if (has_file_data && jcr->crypto.pki_encrypt) {
488 if (!crypto_session_send(jcr, sd)) {
494 * Open any file with data that we intend to save, then save it.
496 * Note, if is_win32_backup, we must open the Directory so that
497 * the BackupRead will save its permissions and ownership streams.
499 if (ff_pkt->type != FT_LNKSAVED && S_ISREG(ff_pkt->statp.st_mode)) {
501 do_read = !is_portable_backup(&ff_pkt->bfd) || ff_pkt->statp.st_size > 0;
503 do_read = ff_pkt->statp.st_size > 0;
505 } else if (ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO ||
506 ff_pkt->type == FT_REPARSE ||
507 (!is_portable_backup(&ff_pkt->bfd) && ff_pkt->type == FT_DIREND)) {
510 if (ff_pkt->cmd_plugin) {
514 Dmsg1(100, "do_read=%d\n", do_read);
518 if (ff_pkt->type == FT_FIFO) {
519 tid = start_thread_timer(jcr, pthread_self(), 60);
523 int noatime = ff_pkt->flags & FO_NOATIME ? O_NOATIME : 0;
524 ff_pkt->bfd.reparse_point = ff_pkt->type == FT_REPARSE;
525 if (bopen(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY | noatime, 0) < 0) {
526 ff_pkt->ff_errno = errno;
528 Jmsg(jcr, M_NOTSAVED, 0, _(" Cannot open \"%s\": ERR=%s.\n"), ff_pkt->fname,
532 stop_thread_timer(tid);
538 stop_thread_timer(tid);
542 stat = send_data(jcr, data_stream, ff_pkt, digest, signing_digest);
544 if (ff_pkt->flags & FO_CHKCHANGES) {
545 has_file_changed(jcr, ff_pkt);
548 bclose(&ff_pkt->bfd);
555 #ifdef HAVE_DARWIN_OS
556 /* Regular files can have resource forks and Finder Info */
557 if (ff_pkt->type != FT_LNKSAVED && (S_ISREG(ff_pkt->statp.st_mode) &&
558 ff_pkt->flags & FO_HFSPLUS)) {
559 if (ff_pkt->hfsinfo.rsrclength > 0) {
562 if (!bopen_rsrc(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0) < 0) {
563 ff_pkt->ff_errno = errno;
565 Jmsg(jcr, M_NOTSAVED, -1, _(" Cannot open resource fork for \"%s\": ERR=%s.\n"),
566 ff_pkt->fname, be.bstrerror());
568 if (is_bopen(&ff_pkt->bfd)) {
569 bclose(&ff_pkt->bfd);
573 flags = ff_pkt->flags;
574 ff_pkt->flags &= ~(FO_GZIP|FO_SPARSE);
575 if (flags & FO_ENCRYPT) {
576 rsrc_stream = STREAM_ENCRYPTED_MACOS_FORK_DATA;
578 rsrc_stream = STREAM_MACOS_FORK_DATA;
580 stat = send_data(jcr, rsrc_stream, ff_pkt, digest, signing_digest);
581 ff_pkt->flags = flags;
582 bclose(&ff_pkt->bfd);
588 Dmsg1(300, "Saving Finder Info for \"%s\"\n", ff_pkt->fname);
589 sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_HFSPLUS_ATTRIBUTES);
590 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
591 memcpy(sd->msg, ff_pkt->hfsinfo.fndrinfo, 32);
594 crypto_digest_update(digest, (uint8_t *)sd->msg, sd->msglen);
596 if (signing_digest) {
597 crypto_digest_update(signing_digest, (uint8_t *)sd->msg, sd->msglen);
600 sd->signal(BNET_EOD);
604 if (ff_pkt->flags & FO_ACL) {
605 /* Read access ACLs for files, dirs and links */
606 if (!read_and_send_acl(jcr, BACL_TYPE_ACCESS, STREAM_UNIX_ACCESS_ACL)) {
609 /* Directories can have default ACLs too */
610 if (ff_pkt->type == FT_DIREND && (BACL_CAP & BACL_CAP_DEFAULTS_DIR)) {
611 if (!read_and_send_acl(jcr, BACL_TYPE_DEFAULT, STREAM_UNIX_DEFAULT_ACL)) {
617 /* Terminate the signing digest and send it to the Storage daemon */
618 if (signing_digest) {
621 if ((sig = crypto_sign_new(jcr)) == NULL) {
622 Jmsg(jcr, M_FATAL, 0, _("Failed to allocate memory for crypto signature.\n"));
626 if (!crypto_sign_add_signer(sig, signing_digest, jcr->crypto.pki_keypair)) {
627 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
631 /* Get signature size */
632 if (!crypto_sign_encode(sig, NULL, &size)) {
633 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
637 /* Grow the bsock buffer to fit our message if necessary */
638 if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
639 sd->msg = realloc_pool_memory(sd->msg, size);
642 /* Send our header */
643 sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_SIGNED_DIGEST);
644 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
646 /* Encode signature data */
647 if (!crypto_sign_encode(sig, (uint8_t *)sd->msg, &size)) {
648 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
654 sd->signal(BNET_EOD); /* end of checksum */
657 /* Terminate any digest and send it to Storage daemon */
661 sd->fsend("%ld %d 0", jcr->JobFiles, digest_stream);
662 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
664 size = CRYPTO_DIGEST_MAX_SIZE;
666 /* Grow the bsock buffer to fit our message if necessary */
667 if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
668 sd->msg = realloc_pool_memory(sd->msg, size);
671 if (!crypto_digest_finalize(digest, (uint8_t *)sd->msg, &size)) {
672 Jmsg(jcr, M_FATAL, 0, _("An error occurred finalizing signing the stream.\n"));
678 sd->signal(BNET_EOD); /* end of checksum */
680 if (ff_pkt->cmd_plugin) {
681 send_plugin_name(jcr, sd, false); /* signal end of plugin data */
685 rtnstat = 1; /* good return */
689 crypto_digest_free(digest);
691 if (signing_digest) {
692 crypto_digest_free(signing_digest);
695 crypto_sign_free(sig);
701 * Send data read from an already open file descriptor.
703 * We return 1 on sucess and 0 on errors.
706 * We use ff_pkt->statp.st_size when FO_SPARSE to know when to stop
708 * Currently this is not a problem as the only other stream, resource forks,
709 * are not handled as sparse files.
711 int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest,
712 DIGEST *signing_digest)
714 BSOCK *sd = jcr->store_bsock;
715 uint64_t fileAddr = 0; /* file address */
717 int32_t rsize = jcr->buf_size; /* read buffer size */
719 CIPHER_CONTEXT *cipher_ctx = NULL; /* Quell bogus uninitialized warnings */
720 const uint8_t *cipher_input;
721 uint32_t cipher_input_len;
722 uint32_t cipher_block_size;
723 uint32_t encrypted_len;
724 #ifdef FD_NO_SEND_TEST
729 rbuf = sd->msg; /* read buffer */
730 wbuf = sd->msg; /* write buffer */
731 cipher_input = (uint8_t *)rbuf; /* encrypt uncompressed data */
733 Dmsg1(300, "Saving data, type=%d\n", ff_pkt->type);
736 uLong compress_len = 0;
737 uLong max_compress_len = 0;
738 const Bytef *cbuf = NULL;
741 if (ff_pkt->flags & FO_GZIP) {
742 if (ff_pkt->flags & FO_SPARSE) {
743 cbuf = (Bytef *)jcr->compress_buf + SPARSE_FADDR_SIZE;
744 max_compress_len = jcr->compress_buf_size - SPARSE_FADDR_SIZE;
746 cbuf = (Bytef *)jcr->compress_buf;
747 max_compress_len = jcr->compress_buf_size; /* set max length */
749 wbuf = jcr->compress_buf; /* compressed output here */
750 cipher_input = (uint8_t *)jcr->compress_buf; /* encrypt compressed data */
753 * Only change zlib parameters if there is no pending operation.
754 * This should never happen as deflatereset is called after each
758 if (((z_stream*)jcr->pZLIB_compress_workset)->total_in == 0) {
759 /* set gzip compression level - must be done per file */
760 if ((zstat=deflateParams((z_stream*)jcr->pZLIB_compress_workset,
761 ff_pkt->GZIP_level, Z_DEFAULT_STRATEGY)) != Z_OK) {
762 Jmsg(jcr, M_FATAL, 0, _("Compression deflateParams error: %d\n"), zstat);
763 set_jcr_job_status(jcr, JS_ErrorTerminated);
769 const uint32_t max_compress_len = 0;
772 if (ff_pkt->flags & FO_ENCRYPT) {
773 if (ff_pkt->flags & FO_SPARSE) {
774 Jmsg0(jcr, M_FATAL, 0, _("Encrypting sparse data not supported.\n"));
777 /* Allocate the cipher context */
778 if ((cipher_ctx = crypto_cipher_new(jcr->crypto.pki_session, true,
779 &cipher_block_size)) == NULL) {
780 /* Shouldn't happen! */
781 Jmsg0(jcr, M_FATAL, 0, _("Failed to initialize encryption context.\n"));
786 * Grow the crypto buffer, if necessary.
787 * crypto_cipher_update() will buffer up to (cipher_block_size - 1).
788 * We grow crypto_buf to the maximum number of blocks that
789 * could be returned for the given read buffer size.
790 * (Using the larger of either rsize or max_compress_len)
792 jcr->crypto.crypto_buf = check_pool_memory_size(jcr->crypto.crypto_buf,
793 (MAX(rsize + (int)sizeof(uint32_t), (int32_t)max_compress_len) +
794 cipher_block_size - 1) / cipher_block_size * cipher_block_size);
796 wbuf = jcr->crypto.crypto_buf; /* Encrypted, possibly compressed output here. */
800 * Send Data header to Storage daemon
801 * <file-index> <stream> <info>
803 if (!sd->fsend("%ld %d 0", jcr->JobFiles, stream)) {
804 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
808 Dmsg1(300, ">stored: datahdr %s\n", sd->msg);
811 * Make space at beginning of buffer for fileAddr because this
812 * same buffer will be used for writing if compression is off.
814 if (ff_pkt->flags & FO_SPARSE) {
815 rbuf += SPARSE_FADDR_SIZE;
816 rsize -= SPARSE_FADDR_SIZE;
817 #ifdef HAVE_FREEBSD_OS
819 * To read FreeBSD partitions, the read size must be
822 rsize = (rsize/512) * 512;
826 /* a RAW device read on win32 only works if the buffer is a multiple of 512 */
828 if (S_ISBLK(ff_pkt->statp.st_mode))
829 rsize = (rsize/512) * 512;
835 while ((sd->msglen=(uint32_t)bread(&ff_pkt->bfd, rbuf, rsize)) > 0) {
837 /* Check for sparse blocks */
838 if (ff_pkt->flags & FO_SPARSE) {
840 bool haveBlock = true;
841 if (sd->msglen == rsize &&
842 fileAddr+sd->msglen < (uint64_t)ff_pkt->statp.st_size ||
843 ((ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO) &&
844 (uint64_t)ff_pkt->statp.st_size == 0)) {
845 haveBlock = !is_buf_zero(rbuf, rsize);
848 ser_begin(wbuf, SPARSE_FADDR_SIZE);
849 ser_uint64(fileAddr); /* store fileAddr in begin of buffer */
851 fileAddr += sd->msglen; /* update file address */
853 continue; /* skip block of zeros */
857 jcr->ReadBytes += sd->msglen; /* count bytes read */
859 /* Uncompressed cipher input length */
860 cipher_input_len = sd->msglen;
862 /* Update checksum if requested */
864 crypto_digest_update(digest, (uint8_t *)rbuf, sd->msglen);
867 /* Update signing digest if requested */
868 if (signing_digest) {
869 crypto_digest_update(signing_digest, (uint8_t *)rbuf, sd->msglen);
873 /* Do compression if turned on */
874 if (ff_pkt->flags & FO_GZIP && jcr->pZLIB_compress_workset) {
875 Dmsg3(400, "cbuf=0x%x rbuf=0x%x len=%u\n", cbuf, rbuf, sd->msglen);
877 ((z_stream*)jcr->pZLIB_compress_workset)->next_in = (Bytef *)rbuf;
878 ((z_stream*)jcr->pZLIB_compress_workset)->avail_in = sd->msglen;
879 ((z_stream*)jcr->pZLIB_compress_workset)->next_out = (Bytef *)cbuf;
880 ((z_stream*)jcr->pZLIB_compress_workset)->avail_out = max_compress_len;
882 if ((zstat=deflate((z_stream*)jcr->pZLIB_compress_workset, Z_FINISH)) != Z_STREAM_END) {
883 Jmsg(jcr, M_FATAL, 0, _("Compression deflate error: %d\n"), zstat);
884 set_jcr_job_status(jcr, JS_ErrorTerminated);
887 compress_len = ((z_stream*)jcr->pZLIB_compress_workset)->total_out;
888 /* reset zlib stream to be able to begin from scratch again */
889 if ((zstat=deflateReset((z_stream*)jcr->pZLIB_compress_workset)) != Z_OK) {
890 Jmsg(jcr, M_FATAL, 0, _("Compression deflateReset error: %d\n"), zstat);
891 set_jcr_job_status(jcr, JS_ErrorTerminated);
895 Dmsg2(400, "compressed len=%d uncompressed len=%d\n", compress_len,
898 sd->msglen = compress_len; /* set compressed length */
899 cipher_input_len = compress_len;
903 * Note, here we prepend the current record length to the beginning
904 * of the encrypted data. This is because both sparse and compression
905 * restore handling want records returned to them with exactly the
906 * same number of bytes that were processed in the backup handling.
907 * That is, both are block filters rather than a stream. When doing
908 * compression, the compression routines may buffer data, so that for
909 * any one record compressed, when it is decompressed the same size
910 * will not be obtained. Of course, the buffered data eventually comes
911 * out in subsequent crypto_cipher_update() calls or at least
912 * when crypto_cipher_finalize() is called. Unfortunately, this
913 * "feature" of encryption enormously complicates the restore code.
915 if (ff_pkt->flags & FO_ENCRYPT) {
916 uint32_t initial_len = 0;
919 if (ff_pkt->flags & FO_SPARSE) {
920 cipher_input_len += SPARSE_FADDR_SIZE;
923 /* Encrypt the length of the input block */
924 uint8_t packet_len[sizeof(uint32_t)];
926 ser_begin(packet_len, sizeof(uint32_t));
927 ser_uint32(cipher_input_len); /* store data len in begin of buffer */
928 Dmsg1(20, "Encrypt len=%d\n", cipher_input_len);
930 if (!crypto_cipher_update(cipher_ctx, packet_len, sizeof(packet_len),
931 (uint8_t *)jcr->crypto.crypto_buf, &initial_len)) {
932 /* Encryption failed. Shouldn't happen. */
933 Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
937 /* Encrypt the input block */
938 if (crypto_cipher_update(cipher_ctx, cipher_input, cipher_input_len,
939 (uint8_t *)&jcr->crypto.crypto_buf[initial_len], &encrypted_len)) {
940 if ((initial_len + encrypted_len) == 0) {
941 /* No full block of data available, read more data */
944 Dmsg2(400, "encrypted len=%d unencrypted len=%d\n", encrypted_len,
946 sd->msglen = initial_len + encrypted_len; /* set encrypted length */
948 /* Encryption failed. Shouldn't happen. */
949 Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
954 /* Send the buffer to the Storage daemon */
955 if (ff_pkt->flags & FO_SPARSE) {
956 sd->msglen += SPARSE_FADDR_SIZE; /* include fileAddr in size */
958 sd->msg = wbuf; /* set correct write buffer */
960 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
964 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
966 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
967 sd->msg = msgsave; /* restore read buffer */
969 } /* end while read file data */
971 if (sd->msglen < 0) { /* error */
973 Jmsg(jcr, M_ERROR, 0, _("Read error on file %s. ERR=%s\n"),
974 ff_pkt->fname, be.bstrerror(ff_pkt->bfd.berrno));
975 if (jcr->Errors++ > 1000) { /* insanity check */
976 Jmsg(jcr, M_FATAL, 0, _("Too many errors.\n"));
978 } else if (ff_pkt->flags & FO_ENCRYPT) {
980 * For encryption, we must call finalize to push out any
983 if (!crypto_cipher_finalize(cipher_ctx, (uint8_t *)jcr->crypto.crypto_buf,
985 /* Padding failed. Shouldn't happen. */
986 Jmsg(jcr, M_FATAL, 0, _("Encryption padding error\n"));
990 /* Note, on SSL pre-0.9.7, there is always some output */
991 if (encrypted_len > 0) {
992 sd->msglen = encrypted_len; /* set encrypted length */
993 sd->msg = jcr->crypto.crypto_buf; /* set correct write buffer */
995 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
999 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
1000 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
1001 sd->msg = msgsave; /* restore bnet buffer */
1005 if (!sd->signal(BNET_EOD)) { /* indicate end of file data */
1006 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1011 /* Free the cipher context */
1013 crypto_cipher_free(cipher_ctx);
1018 /* Free the cipher context */
1020 crypto_cipher_free(cipher_ctx);
1023 sd->msg = msgsave; /* restore bnet buffer */
1029 * Read and send an ACL for the last encountered file.
1031 static bool read_and_send_acl(JCR *jcr, int acltype, int stream)
1034 BSOCK *sd = jcr->store_bsock;
1037 #ifdef FD_NO_SEND_TEST
1041 len = bacl_get(jcr, acltype);
1043 Jmsg1(jcr, M_WARNING, 0, _("Error reading ACL of %s\n"), jcr->last_fname);
1047 return true; /* no ACL */
1051 if (!sd->fsend("%ld %d 0", jcr->JobFiles, stream)) {
1052 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1057 /* Send the buffer to the storage deamon */
1058 Dmsg2(400, "Backing up ACL type 0x%2x <%s>\n", acltype, jcr->acl_text);
1060 sd->msg = jcr->acl_text;
1061 sd->msglen = len + 1;
1065 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1070 jcr->JobBytes += sd->msglen;
1072 if (!sd->signal(BNET_EOD)) {
1073 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1078 Dmsg1(200, "ACL of file: %s successfully backed up!\n", jcr->last_fname);
1083 bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream)
1085 BSOCK *sd = jcr->store_bsock;
1086 char attribs[MAXSTRING];
1087 char attribsEx[MAXSTRING];
1090 #ifdef FD_NO_SEND_TEST
1094 Dmsg1(300, "encode_and_send_attrs fname=%s\n", ff_pkt->fname);
1095 /* Find what data stream we will use, then encode the attributes */
1096 if ((data_stream = select_data_stream(ff_pkt)) == STREAM_NONE) {
1097 /* This should not happen */
1098 Jmsg0(jcr, M_FATAL, 0, _("Invalid file flags, no supported data stream type.\n"));
1101 encode_stat(attribs, ff_pkt, data_stream);
1103 /* Now possibly extend the attributes */
1104 attr_stream = encode_attribsEx(jcr, attribsEx, ff_pkt);
1106 Dmsg3(300, "File %s\nattribs=%s\nattribsEx=%s\n", ff_pkt->fname, attribs, attribsEx);
1109 jcr->JobFiles++; /* increment number of files sent */
1110 ff_pkt->FileIndex = jcr->JobFiles; /* return FileIndex */
1111 pm_strcpy(jcr->last_fname, ff_pkt->fname);
1115 * Send Attributes header to Storage daemon
1116 * <file-index> <stream> <info>
1118 if (!sd->fsend("%ld %d 0", jcr->JobFiles, attr_stream)) {
1119 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1123 Dmsg1(300, ">stored: attrhdr %s\n", sd->msg);
1126 * Send file attributes to Storage daemon
1129 * Filename (full path)
1130 * Encoded attributes
1131 * Link name (if type==FT_LNK or FT_LNKSAVED)
1132 * Encoded extended-attributes (for Win32)
1134 * For a directory, link is the same as fname, but with trailing
1135 * slash. For a linked file, link is the link.
1137 if (ff_pkt->type != FT_DELETED) { /* already stripped */
1140 if (ff_pkt->type == FT_LNK || ff_pkt->type == FT_LNKSAVED) {
1141 Dmsg2(300, "Link %s to %s\n", ff_pkt->fname, ff_pkt->link);
1142 stat = sd->fsend("%ld %d %s%c%s%c%s%c%s%c", jcr->JobFiles,
1143 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, ff_pkt->link, 0,
1145 } else if (ff_pkt->type == FT_DIREND || ff_pkt->type == FT_REPARSE) {
1146 /* Here link is the canonical filename (i.e. with trailing slash) */
1147 stat = sd->fsend("%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
1148 ff_pkt->type, ff_pkt->link, 0, attribs, 0, 0, attribsEx, 0);
1150 stat = sd->fsend("%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
1151 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, 0, attribsEx, 0);
1153 if (ff_pkt->type != FT_DELETED) {
1154 unstrip_path(ff_pkt);
1157 Dmsg2(300, ">stored: attr len=%d: %s\n", sd->msglen, sd->msg);
1159 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1163 sd->signal(BNET_EOD); /* indicate end of attributes data */
1168 * Do in place strip of path
1170 static bool do_strip(int count, char *in)
1176 /* Copy to first path separator -- Win32 might have c: ... */
1177 while (*in && !IsPathSeparator(*in)) {
1181 numsep++; /* one separator seen */
1182 for (stripped=0; stripped<count && *in; stripped++) {
1183 while (*in && !IsPathSeparator(*in)) {
1184 in++; /* skip chars */
1187 numsep++; /* count separators seen */
1188 in++; /* skip separator */
1192 while (*in) { /* copy to end */
1193 if (IsPathSeparator(*in)) {
1199 Dmsg4(500, "stripped=%d count=%d numsep=%d sep>count=%d\n",
1200 stripped, count, numsep, numsep>count);
1201 return stripped==count && numsep>count;
1205 * If requested strip leading components of the path so that we can
1206 * save file as if it came from a subdirectory. This is most useful
1207 * for dealing with snapshots, by removing the snapshot directory, or
1208 * in handling vendor migrations where files have been restored with
1209 * a vendor product into a subdirectory.
1211 void strip_path(FF_PKT *ff_pkt)
1213 if (!(ff_pkt->flags & FO_STRIPPATH) || ff_pkt->strip_path <= 0) {
1214 Dmsg1(200, "No strip for %s\n", ff_pkt->fname);
1217 if (!ff_pkt->fname_save) {
1218 ff_pkt->fname_save = get_pool_memory(PM_FNAME);
1219 ff_pkt->link_save = get_pool_memory(PM_FNAME);
1221 pm_strcpy(ff_pkt->fname_save, ff_pkt->fname);
1224 * Strip path. If it doesn't succeed put it back. If
1225 * it does, and there is a different link string,
1226 * attempt to strip the link. If it fails, back them
1228 * Do not strip symlinks.
1229 * I.e. if either stripping fails don't strip anything.
1231 if (do_strip(ff_pkt->strip_path, ff_pkt->fname)) {
1232 /* Strip links but not symlinks */
1233 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1234 pm_strcpy(ff_pkt->link_save, ff_pkt->link);
1235 if (!do_strip(ff_pkt->strip_path, ff_pkt->link)) {
1236 strcpy(ff_pkt->link, ff_pkt->link_save);
1237 strcpy(ff_pkt->fname, ff_pkt->fname_save);
1241 strcpy(ff_pkt->fname, ff_pkt->fname_save);
1243 Dmsg2(200, "fname=%s stripped=%s\n", ff_pkt->fname_save, ff_pkt->fname);
1246 void unstrip_path(FF_PKT *ff_pkt)
1248 if (!(ff_pkt->flags & FO_STRIPPATH) || ff_pkt->strip_path <= 0) {
1251 strcpy(ff_pkt->fname, ff_pkt->fname_save);
1252 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1253 strcpy(ff_pkt->link, ff_pkt->link_save);