2 Bacula® - The Network Backup Solution
4 Copyright (C) 2000-2007 Free Software Foundation Europe e.V.
6 The main author of Bacula is Kern Sibbald, with contributions from
7 many others, a complete list can be found in the file AUTHORS.
8 This program is Free Software; you can redistribute it and/or
9 modify it under the terms of version two of the GNU General Public
10 License as published by the Free Software Foundation and included
13 This program is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
23 Bacula® is a registered trademark of John Walker.
24 The licensor of Bacula is the Free Software Foundation Europe
25 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
26 Switzerland, email:ftf@fsfeurope.org.
29 * Bacula File Daemon backup.c send file attributes and data
30 * to the Storage daemon.
32 * Kern Sibbald, March MM
41 /* Forward referenced functions */
42 static int save_file(FF_PKT *ff_pkt, void *pkt, bool top_level);
43 static void strip_path(FF_PKT *ff_pkt);
44 static void unstrip_path(FF_PKT *ff_pkt);
45 static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, DIGEST *signature_digest);
46 static bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream);
47 static bool read_and_send_acl(JCR *jcr, int acltype, int stream);
50 * Find all the requested files and send them
51 * to the Storage daemon.
53 * Note, we normally carry on a one-way
54 * conversation from this point on with the SD, simply blasting
55 * data to him. To properly know what is going on, we
56 * also run a "heartbeat" monitor which reads the socket and
57 * reacts accordingly (at the moment it has nothing to do
58 * except echo the heartbeat to the Director).
61 bool blast_data_to_storage_daemon(JCR *jcr, char *addr)
65 // TODO landonf: Allow user to specify encryption algorithm
66 crypto_cipher_t cipher = CRYPTO_CIPHER_AES_128_CBC;
69 sd = jcr->store_bsock;
71 set_jcr_job_status(jcr, JS_Running);
73 Dmsg1(300, "bfiled: opened data connection %d to stored\n", sd->m_fd);
76 CLIENT *client = (CLIENT *)GetNextRes(R_CLIENT, NULL);
80 buf_size = client->max_network_buffer_size;
82 buf_size = 0; /* use default */
84 if (!bnet_set_buffer_size(sd, buf_size, BNET_SETBUF_WRITE)) {
85 set_jcr_job_status(jcr, JS_ErrorTerminated);
86 Jmsg(jcr, M_FATAL, 0, _("Cannot set buffer size FD->SD.\n"));
90 jcr->buf_size = sd->msglen;
91 /* Adjust for compression so that output buffer is
92 * 12 bytes + 0.1% larger than input buffer plus 18 bytes.
93 * This gives a bit extra plus room for the sparse addr if any.
94 * Note, we adjust the read size to be smaller so that the
95 * same output buffer can be used without growing it.
97 * The zlib compression workset is initialized here to minimise
98 * the "per file" load. The jcr member is only set, if the init was successful.
100 jcr->compress_buf_size = jcr->buf_size + ((jcr->buf_size+999) / 1000) + 30;
101 jcr->compress_buf = get_memory(jcr->compress_buf_size);
104 z_stream *pZlibStream = (z_stream*)malloc(sizeof(z_stream));
106 pZlibStream->zalloc = Z_NULL;
107 pZlibStream->zfree = Z_NULL;
108 pZlibStream->opaque = Z_NULL;
109 pZlibStream->state = Z_NULL;
111 if (deflateInit(pZlibStream, Z_DEFAULT_COMPRESSION) == Z_OK)
112 jcr->pZLIB_compress_workset = pZlibStream;
118 /* Create encryption session data and a cached, DER-encoded session data
119 * structure. We use a single session key for each backup, so we'll encode
120 * the session data only once. */
121 if (jcr->pki_encrypt) {
124 /* Create per-job session encryption context */
125 jcr->pki_session = crypto_session_new(cipher, jcr->pki_recipients);
127 /* Get the session data size */
128 if (crypto_session_encode(jcr->pki_session, (uint8_t *)0, &size) == false) {
129 Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n"));
133 /* Allocate buffer */
134 jcr->pki_session_encoded = (uint8_t *)malloc(size);
136 /* Encode session data */
137 if (crypto_session_encode(jcr->pki_session, jcr->pki_session_encoded, &size) == false) {
138 Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n"));
142 /* ... and store the encoded size */
143 jcr->pki_session_encoded_size = size;
145 /* Allocate the encryption/decryption buffer */
146 jcr->crypto_buf = get_memory(CRYPTO_CIPHER_MAX_BLOCK_SIZE);
149 Dmsg1(300, "set_find_options ff=%p\n", jcr->ff);
150 set_find_options((FF_PKT *)jcr->ff, jcr->incremental, jcr->mtime);
151 Dmsg0(300, "start find files\n");
153 start_heartbeat_monitor(jcr);
155 jcr->acl_text = get_pool_memory(PM_MESSAGE);
157 /* Subroutine save_file() is called for each file */
158 if (!find_files(jcr, (FF_PKT *)jcr->ff, save_file, (void *)jcr)) {
159 ok = false; /* error */
160 set_jcr_job_status(jcr, JS_ErrorTerminated);
163 free_pool_memory(jcr->acl_text);
165 stop_heartbeat_monitor(jcr);
167 sd->signal(BNET_EOD); /* end of sending data */
173 if (jcr->compress_buf) {
174 free_pool_memory(jcr->compress_buf);
175 jcr->compress_buf = NULL;
177 if (jcr->pZLIB_compress_workset) {
178 /* Free the zlib stream */
180 deflateEnd((z_stream *)jcr->pZLIB_compress_workset);
182 free (jcr->pZLIB_compress_workset);
183 jcr->pZLIB_compress_workset = NULL;
185 if (jcr->crypto_buf) {
186 free_pool_memory(jcr->crypto_buf);
187 jcr->crypto_buf = NULL;
189 if (jcr->pki_session) {
190 crypto_session_free(jcr->pki_session);
192 if (jcr->pki_session_encoded) {
193 free(jcr->pki_session_encoded);
194 jcr->pki_session_encoded = NULL;
197 Dmsg1(100, "end blast_data ok=%d\n", ok);
202 * Called here by find() for each file included.
203 * This is a callback. The original is find_files() above.
205 * Send the file and its data to the Storage daemon.
209 * -1 to ignore file/directory (not used here)
211 static int save_file(FF_PKT *ff_pkt, void *vjcr, bool top_level)
213 bool do_read = false;
214 int stat, data_stream;
216 DIGEST *digest = NULL;
217 DIGEST *signing_digest = NULL;
218 int digest_stream = STREAM_NONE;
219 SIGNATURE *sig = NULL;
220 bool has_file_data = false;
221 // TODO landonf: Allow the user to specify the digest algorithm
223 crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA256;
225 crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA1;
227 JCR *jcr = (JCR *)vjcr;
228 BSOCK *sd = jcr->store_bsock;
230 if (job_canceled(jcr)) {
234 jcr->num_files_examined++; /* bump total file count */
236 switch (ff_pkt->type) {
237 case FT_LNKSAVED: /* Hard linked, file already saved */
238 Dmsg2(130, "FT_LNKSAVED hard link: %s => %s\n", ff_pkt->fname, ff_pkt->link);
241 Dmsg1(130, "FT_REGE saving: %s\n", ff_pkt->fname);
242 has_file_data = true;
245 Dmsg1(130, "FT_REG saving: %s\n", ff_pkt->fname);
246 has_file_data = true;
249 Dmsg2(130, "FT_LNK saving: %s -> %s\n", ff_pkt->fname, ff_pkt->link);
252 jcr->num_files_examined--; /* correct file count */
253 return 1; /* not used */
255 Jmsg(jcr, M_INFO, 1, _(" Recursion turned off. Will not descend from %s into %s\n"),
256 ff_pkt->top_fname, ff_pkt->fname);
257 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
260 /* Suppress message for /dev filesystems */
261 if (strncmp(ff_pkt->fname, "/dev/", 5) != 0) {
262 Jmsg(jcr, M_INFO, 1, _(" %s is a different filesystem. Will not descend from %s into %s\n"),
263 ff_pkt->fname, ff_pkt->top_fname, ff_pkt->fname);
265 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
268 Jmsg(jcr, M_INFO, 1, _(" Disallowed filesystem. Will not descend from %s into %s\n"),
269 ff_pkt->top_fname, ff_pkt->fname);
270 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
273 Jmsg(jcr, M_INFO, 1, _(" Disallowed drive type. Will not descend into %s\n"),
278 Dmsg1(130, "FT_DIREND: %s\n", ff_pkt->link);
281 Dmsg1(130, "FT_SPEC saving: %s\n", ff_pkt->fname);
282 if (S_ISSOCK(ff_pkt->statp.st_mode)) {
283 Jmsg(jcr, M_SKIPPED, 1, _(" Socket file skipped: %s\n"), ff_pkt->fname);
288 Dmsg1(130, "FT_RAW saving: %s\n", ff_pkt->fname);
289 has_file_data = true;
292 Dmsg1(130, "FT_FIFO saving: %s\n", ff_pkt->fname);
296 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not access %s: ERR=%s\n"), ff_pkt->fname,
297 be.bstrerror(ff_pkt->ff_errno));
303 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not follow link %s: ERR=%s\n"),
304 ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
310 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not stat %s: ERR=%s\n"), ff_pkt->fname,
311 be.bstrerror(ff_pkt->ff_errno));
317 Jmsg(jcr, M_SKIPPED, 1, _(" Unchanged file skipped: %s\n"), ff_pkt->fname);
320 Jmsg(jcr, M_NOTSAVED, 0, _(" Archive file not saved: %s\n"), ff_pkt->fname);
324 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not open directory %s: ERR=%s\n"),
325 ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
330 Jmsg(jcr, M_NOTSAVED, 0, _(" Unknown file type %d; not saved: %s\n"),
331 ff_pkt->type, ff_pkt->fname);
336 Dmsg1(130, "bfiled: sending %s to stored\n", ff_pkt->fname);
338 /* Digests and encryption are only useful if there's file data */
341 * Setup for digest handling. If this fails, the digest will be set to NULL
342 * and not used. Note, the digest (file hash) can be any one of the four
345 * The signing digest is a single algorithm depending on
346 * whether or not we have SHA2.
347 * ****FIXME**** the signing algoritm should really be
348 * determined a different way!!!!!! What happens if
349 * sha2 was available during backup but not restore?
351 if (ff_pkt->flags & FO_MD5) {
352 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_MD5);
353 digest_stream = STREAM_MD5_DIGEST;
355 } else if (ff_pkt->flags & FO_SHA1) {
356 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA1);
357 digest_stream = STREAM_SHA1_DIGEST;
359 } else if (ff_pkt->flags & FO_SHA256) {
360 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA256);
361 digest_stream = STREAM_SHA256_DIGEST;
363 } else if (ff_pkt->flags & FO_SHA512) {
364 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA512);
365 digest_stream = STREAM_SHA512_DIGEST;
368 /* Did digest initialization fail? */
369 if (digest_stream != STREAM_NONE && digest == NULL) {
370 Jmsg(jcr, M_WARNING, 0, _("%s digest initialization failed\n"),
371 stream_to_ascii(digest_stream));
375 * Set up signature digest handling. If this fails, the signature digest will be set to
378 // TODO landonf: We should really only calculate the digest once, for both verification and signing.
380 signing_digest = crypto_digest_new(jcr, signing_algorithm);
382 /* Full-stop if a failure occurred initializing the signature digest */
383 if (signing_digest == NULL) {
384 Jmsg(jcr, M_NOTSAVED, 0, _("%s signature digest initialization failed\n"),
385 stream_to_ascii(signing_algorithm));
391 /* Enable encryption */
392 if (jcr->pki_encrypt) {
393 ff_pkt->flags |= FO_ENCRYPT;
397 /* Initialise the file descriptor we use for data and other streams. */
399 if (ff_pkt->flags & FO_PORTABLE) {
400 set_portable_backup(&ff_pkt->bfd); /* disable Win32 BackupRead() */
402 if (ff_pkt->reader) {
403 if (!set_prog(&ff_pkt->bfd, ff_pkt->reader, jcr)) {
404 Jmsg(jcr, M_FATAL, 0, _("Python reader program \"%s\" not found.\n"),
410 /* Send attributes -- must be done after binit() */
411 if (!encode_and_send_attributes(jcr, ff_pkt, data_stream)) {
415 /* Set up the encryption context and send the session data to the SD */
416 if (has_file_data && jcr->pki_encrypt) {
417 /* Send our header */
418 Dmsg2(100, "Send hdr fi=%ld stream=%d\n", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA);
419 sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA);
421 /* Grow the bsock buffer to fit our message if necessary */
422 if (sizeof_pool_memory(sd->msg) < jcr->pki_session_encoded_size) {
423 sd->msg = realloc_pool_memory(sd->msg, jcr->pki_session_encoded_size);
426 /* Copy our message over and send it */
427 memcpy(sd->msg, jcr->pki_session_encoded, jcr->pki_session_encoded_size);
428 sd->msglen = jcr->pki_session_encoded_size;
429 jcr->JobBytes += sd->msglen;
431 Dmsg1(100, "Send data len=%d\n", sd->msglen);
433 sd->signal(BNET_EOD);
437 * Open any file with data that we intend to save, then save it.
439 * Note, if is_win32_backup, we must open the Directory so that
440 * the BackupRead will save its permissions and ownership streams.
442 if (ff_pkt->type != FT_LNKSAVED && S_ISREG(ff_pkt->statp.st_mode)) {
444 do_read = !is_portable_backup(&ff_pkt->bfd) || ff_pkt->statp.st_size > 0;
446 do_read = ff_pkt->statp.st_size > 0;
448 } else if (ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO ||
449 ff_pkt->type == FT_REPARSE ||
450 (!is_portable_backup(&ff_pkt->bfd) && ff_pkt->type == FT_DIREND)) {
456 if (ff_pkt->type == FT_FIFO) {
457 tid = start_thread_timer(jcr, pthread_self(), 60);
461 int noatime = ff_pkt->flags & FO_NOATIME ? O_NOATIME : 0;
462 ff_pkt->bfd.reparse_point = ff_pkt->type == FT_REPARSE;
463 if (bopen(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY | noatime, 0) < 0) {
464 ff_pkt->ff_errno = errno;
466 Jmsg(jcr, M_NOTSAVED, 0, _(" Cannot open %s: ERR=%s.\n"), ff_pkt->fname,
470 stop_thread_timer(tid);
476 stop_thread_timer(tid);
480 stat = send_data(jcr, data_stream, ff_pkt, digest, signing_digest);
482 if (ff_pkt->flags & FO_CHKCHANGES) {
483 has_file_changed(jcr, ff_pkt);
486 bclose(&ff_pkt->bfd);
493 #ifdef HAVE_DARWIN_OS
494 /* Regular files can have resource forks and Finder Info */
495 if (ff_pkt->type != FT_LNKSAVED && (S_ISREG(ff_pkt->statp.st_mode) &&
496 ff_pkt->flags & FO_HFSPLUS)) {
497 if (ff_pkt->hfsinfo.rsrclength > 0) {
500 if (!bopen_rsrc(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0) < 0) {
501 ff_pkt->ff_errno = errno;
503 Jmsg(jcr, M_NOTSAVED, -1, _(" Cannot open resource fork for %s: ERR=%s.\n"),
504 ff_pkt->fname, be.bstrerror());
506 if (is_bopen(&ff_pkt->bfd)) {
507 bclose(&ff_pkt->bfd);
511 flags = ff_pkt->flags;
512 ff_pkt->flags &= ~(FO_GZIP|FO_SPARSE);
513 if (flags & FO_ENCRYPT) {
514 rsrc_stream = STREAM_ENCRYPTED_MACOS_FORK_DATA;
516 rsrc_stream = STREAM_MACOS_FORK_DATA;
518 stat = send_data(jcr, rsrc_stream, ff_pkt, digest, signing_digest);
519 ff_pkt->flags = flags;
520 bclose(&ff_pkt->bfd);
526 Dmsg1(300, "Saving Finder Info for \"%s\"\n", ff_pkt->fname);
527 sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_HFSPLUS_ATTRIBUTES);
528 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
529 memcpy(sd->msg, ff_pkt->hfsinfo.fndrinfo, 32);
532 crypto_digest_update(digest, (uint8_t *)sd->msg, sd->msglen);
534 if (signing_digest) {
535 crypto_digest_update(signing_digest, (uint8_t *)sd->msg, sd->msglen);
538 sd->signal(BNET_EOD);
542 if (ff_pkt->flags & FO_ACL) {
543 /* Read access ACLs for files, dirs and links */
544 if (!read_and_send_acl(jcr, BACL_TYPE_ACCESS, STREAM_UNIX_ATTRIBUTES_ACCESS_ACL)) {
547 /* Directories can have default ACLs too */
548 if (ff_pkt->type == FT_DIREND && (BACL_CAP & BACL_CAP_DEFAULTS_DIR)) {
549 if (!read_and_send_acl(jcr, BACL_TYPE_DEFAULT, STREAM_UNIX_ATTRIBUTES_DEFAULT_ACL)) {
555 /* Terminate the signing digest and send it to the Storage daemon */
556 if (signing_digest) {
559 if ((sig = crypto_sign_new(jcr)) == NULL) {
560 Jmsg(jcr, M_FATAL, 0, _("Failed to allocate memory for crypto signature.\n"));
564 if (!crypto_sign_add_signer(sig, signing_digest, jcr->pki_keypair)) {
565 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
569 /* Get signature size */
570 if (!crypto_sign_encode(sig, NULL, &size)) {
571 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
575 /* Grow the bsock buffer to fit our message if necessary */
576 if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
577 sd->msg = realloc_pool_memory(sd->msg, size);
580 /* Send our header */
581 sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_SIGNED_DIGEST);
582 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
584 /* Encode signature data */
585 if (!crypto_sign_encode(sig, (uint8_t *)sd->msg, &size)) {
586 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
592 sd->signal(BNET_EOD); /* end of checksum */
595 /* Terminate any digest and send it to Storage daemon and the Director */
599 sd->fsend("%ld %d 0", jcr->JobFiles, digest_stream);
600 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
602 size = CRYPTO_DIGEST_MAX_SIZE;
604 /* Grow the bsock buffer to fit our message if necessary */
605 if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
606 sd->msg = realloc_pool_memory(sd->msg, size);
609 if (!crypto_digest_finalize(digest, (uint8_t *)sd->msg, &size)) {
610 Jmsg(jcr, M_FATAL, 0, _("An error occurred finalizing signing the stream.\n"));
616 sd->signal(BNET_EOD); /* end of checksum */
620 rtnstat = 1; /* good return */
624 crypto_digest_free(digest);
626 if (signing_digest) {
627 crypto_digest_free(signing_digest);
630 crypto_sign_free(sig);
636 * Send data read from an already open file descriptor.
638 * We return 1 on sucess and 0 on errors.
641 * We use ff_pkt->statp.st_size when FO_SPARSE to know when to stop
643 * Currently this is not a problem as the only other stream, resource forks,
644 * are not handled as sparse files.
646 int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest,
647 DIGEST *signing_digest)
649 BSOCK *sd = jcr->store_bsock;
650 uint64_t fileAddr = 0; /* file address */
652 int32_t rsize = jcr->buf_size; /* read buffer size */
654 CIPHER_CONTEXT *cipher_ctx = NULL; /* Quell bogus uninitialized warnings */
655 const uint8_t *cipher_input;
656 uint32_t cipher_input_len;
657 uint32_t cipher_block_size;
658 uint32_t encrypted_len;
659 #ifdef FD_NO_SEND_TEST
664 rbuf = sd->msg; /* read buffer */
665 wbuf = sd->msg; /* write buffer */
666 cipher_input = (uint8_t *)rbuf; /* encrypt uncompressed data */
668 Dmsg1(300, "Saving data, type=%d\n", ff_pkt->type);
671 uLong compress_len = 0;
672 uLong max_compress_len = 0;
673 const Bytef *cbuf = NULL;
676 if (ff_pkt->flags & FO_GZIP) {
677 if (ff_pkt->flags & FO_SPARSE) {
678 cbuf = (Bytef *)jcr->compress_buf + SPARSE_FADDR_SIZE;
679 max_compress_len = jcr->compress_buf_size - SPARSE_FADDR_SIZE;
681 cbuf = (Bytef *)jcr->compress_buf;
682 max_compress_len = jcr->compress_buf_size; /* set max length */
684 wbuf = jcr->compress_buf; /* compressed output here */
685 cipher_input = (uint8_t *)jcr->compress_buf; /* encrypt compressed data */
688 * Only change zlib parameters if there is no pending operation.
689 * This should never happen as deflatereset is called after each
693 if (((z_stream*)jcr->pZLIB_compress_workset)->total_in == 0) {
694 /* set gzip compression level - must be done per file */
695 if ((zstat=deflateParams((z_stream*)jcr->pZLIB_compress_workset,
696 ff_pkt->GZIP_level, Z_DEFAULT_STRATEGY)) != Z_OK) {
697 Jmsg(jcr, M_FATAL, 0, _("Compression deflateParams error: %d\n"), zstat);
698 set_jcr_job_status(jcr, JS_ErrorTerminated);
704 const uint32_t max_compress_len = 0;
707 if (ff_pkt->flags & FO_ENCRYPT) {
708 if (ff_pkt->flags & FO_SPARSE) {
709 Jmsg0(jcr, M_FATAL, 0, _("Encrypting sparse data not supported.\n"));
712 /* Allocate the cipher context */
713 if ((cipher_ctx = crypto_cipher_new(jcr->pki_session, true,
714 &cipher_block_size)) == NULL) {
715 /* Shouldn't happen! */
716 Jmsg0(jcr, M_FATAL, 0, _("Failed to initialize encryption context.\n"));
721 * Grow the crypto buffer, if necessary.
722 * crypto_cipher_update() will buffer up to (cipher_block_size - 1).
723 * We grow crypto_buf to the maximum number of blocks that
724 * could be returned for the given read buffer size.
725 * (Using the larger of either rsize or max_compress_len)
727 jcr->crypto_buf = check_pool_memory_size(jcr->crypto_buf,
728 (MAX(rsize + (int)sizeof(uint32_t), (int32_t)max_compress_len) +
729 cipher_block_size - 1) / cipher_block_size * cipher_block_size);
731 wbuf = jcr->crypto_buf; /* Encrypted, possibly compressed output here. */
735 * Send Data header to Storage daemon
736 * <file-index> <stream> <info>
738 if (!sd->fsend("%ld %d 0", jcr->JobFiles, stream)) {
739 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
743 Dmsg1(300, ">stored: datahdr %s\n", sd->msg);
746 * Make space at beginning of buffer for fileAddr because this
747 * same buffer will be used for writing if compression is off.
749 if (ff_pkt->flags & FO_SPARSE) {
750 rbuf += SPARSE_FADDR_SIZE;
751 rsize -= SPARSE_FADDR_SIZE;
752 #ifdef HAVE_FREEBSD_OS
754 * To read FreeBSD partitions, the read size must be
757 rsize = (rsize/512) * 512;
761 /* a RAW device read on win32 only works if the buffer is a multiple of 512 */
763 if (S_ISBLK(ff_pkt->statp.st_mode))
764 rsize = (rsize/512) * 512;
770 while ((sd->msglen=(uint32_t)bread(&ff_pkt->bfd, rbuf, rsize)) > 0) {
772 /* Check for sparse blocks */
773 if (ff_pkt->flags & FO_SPARSE) {
775 bool haveBlock = true;
776 if (sd->msglen == rsize &&
777 fileAddr+sd->msglen < (uint64_t)ff_pkt->statp.st_size ||
778 ((ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO) &&
779 (uint64_t)ff_pkt->statp.st_size == 0)) {
780 haveBlock = !is_buf_zero(rbuf, rsize);
783 ser_begin(wbuf, SPARSE_FADDR_SIZE);
784 ser_uint64(fileAddr); /* store fileAddr in begin of buffer */
786 fileAddr += sd->msglen; /* update file address */
788 continue; /* skip block of zeros */
792 jcr->ReadBytes += sd->msglen; /* count bytes read */
794 /* Uncompressed cipher input length */
795 cipher_input_len = sd->msglen;
797 /* Update checksum if requested */
799 crypto_digest_update(digest, (uint8_t *)rbuf, sd->msglen);
802 /* Update signing digest if requested */
803 if (signing_digest) {
804 crypto_digest_update(signing_digest, (uint8_t *)rbuf, sd->msglen);
808 /* Do compression if turned on */
809 if (ff_pkt->flags & FO_GZIP && jcr->pZLIB_compress_workset) {
810 Dmsg3(400, "cbuf=0x%x rbuf=0x%x len=%u\n", cbuf, rbuf, sd->msglen);
812 ((z_stream*)jcr->pZLIB_compress_workset)->next_in = (Bytef *)rbuf;
813 ((z_stream*)jcr->pZLIB_compress_workset)->avail_in = sd->msglen;
814 ((z_stream*)jcr->pZLIB_compress_workset)->next_out = (Bytef *)cbuf;
815 ((z_stream*)jcr->pZLIB_compress_workset)->avail_out = max_compress_len;
817 if ((zstat=deflate((z_stream*)jcr->pZLIB_compress_workset, Z_FINISH)) != Z_STREAM_END) {
818 Jmsg(jcr, M_FATAL, 0, _("Compression deflate error: %d\n"), zstat);
819 set_jcr_job_status(jcr, JS_ErrorTerminated);
822 compress_len = ((z_stream*)jcr->pZLIB_compress_workset)->total_out;
823 /* reset zlib stream to be able to begin from scratch again */
824 if ((zstat=deflateReset((z_stream*)jcr->pZLIB_compress_workset)) != Z_OK) {
825 Jmsg(jcr, M_FATAL, 0, _("Compression deflateReset error: %d\n"), zstat);
826 set_jcr_job_status(jcr, JS_ErrorTerminated);
830 Dmsg2(400, "compressed len=%d uncompressed len=%d\n", compress_len,
833 sd->msglen = compress_len; /* set compressed length */
834 cipher_input_len = compress_len;
838 * Note, here we prepend the current record length to the beginning
839 * of the encrypted data. This is because both sparse and compression
840 * restore handling want records returned to them with exactly the
841 * same number of bytes that were processed in the backup handling.
842 * That is, both are block filters rather than a stream. When doing
843 * compression, the compression routines may buffer data, so that for
844 * any one record compressed, when it is decompressed the same size
845 * will not be obtained. Of course, the buffered data eventually comes
846 * out in subsequent crypto_cipher_update() calls or at least
847 * when crypto_cipher_finalize() is called. Unfortunately, this
848 * "feature" of encryption enormously complicates the restore code.
850 if (ff_pkt->flags & FO_ENCRYPT) {
851 uint32_t initial_len = 0;
854 if (ff_pkt->flags & FO_SPARSE) {
855 cipher_input_len += SPARSE_FADDR_SIZE;
858 /* Encrypt the length of the input block */
859 uint8_t packet_len[sizeof(uint32_t)];
861 ser_begin(packet_len, sizeof(uint32_t));
862 ser_uint32(cipher_input_len); /* store data len in begin of buffer */
863 Dmsg1(20, "Encrypt len=%d\n", cipher_input_len);
865 if (!crypto_cipher_update(cipher_ctx, packet_len, sizeof(packet_len),
866 (u_int8_t *)jcr->crypto_buf, &initial_len)) {
867 /* Encryption failed. Shouldn't happen. */
868 Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
872 /* Encrypt the input block */
873 if (crypto_cipher_update(cipher_ctx, cipher_input, cipher_input_len,
874 (u_int8_t *)&jcr->crypto_buf[initial_len], &encrypted_len)) {
875 if ((initial_len + encrypted_len) == 0) {
876 /* No full block of data available, read more data */
879 Dmsg2(400, "encrypted len=%d unencrypted len=%d\n", encrypted_len,
881 sd->msglen = initial_len + encrypted_len; /* set encrypted length */
883 /* Encryption failed. Shouldn't happen. */
884 Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
889 /* Send the buffer to the Storage daemon */
890 if (ff_pkt->flags & FO_SPARSE) {
891 sd->msglen += SPARSE_FADDR_SIZE; /* include fileAddr in size */
893 sd->msg = wbuf; /* set correct write buffer */
895 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
899 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
901 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
902 sd->msg = msgsave; /* restore read buffer */
904 } /* end while read file data */
906 if (sd->msglen < 0) { /* error */
908 Jmsg(jcr, M_ERROR, 0, _("Read error on file %s. ERR=%s\n"),
909 ff_pkt->fname, be.bstrerror(ff_pkt->bfd.berrno));
910 if (jcr->Errors++ > 1000) { /* insanity check */
911 Jmsg(jcr, M_FATAL, 0, _("Too many errors.\n"));
913 } else if (ff_pkt->flags & FO_ENCRYPT) {
915 * For encryption, we must call finalize to push out any
918 if (!crypto_cipher_finalize(cipher_ctx, (uint8_t *)jcr->crypto_buf,
920 /* Padding failed. Shouldn't happen. */
921 Jmsg(jcr, M_FATAL, 0, _("Encryption padding error\n"));
925 /* Note, on SSL pre-0.9.7, there is always some output */
926 if (encrypted_len > 0) {
927 sd->msglen = encrypted_len; /* set encrypted length */
928 sd->msg = jcr->crypto_buf; /* set correct write buffer */
930 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
934 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
935 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
936 sd->msg = msgsave; /* restore bnet buffer */
940 if (!sd->signal(BNET_EOD)) { /* indicate end of file data */
941 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
946 /* Free the cipher context */
948 crypto_cipher_free(cipher_ctx);
953 /* Free the cipher context */
955 crypto_cipher_free(cipher_ctx);
958 sd->msg = msgsave; /* restore bnet buffer */
964 * Read and send an ACL for the last encountered file.
966 static bool read_and_send_acl(JCR *jcr, int acltype, int stream)
969 BSOCK *sd = jcr->store_bsock;
972 #ifdef FD_NO_SEND_TEST
976 len = bacl_get(jcr, acltype);
978 Jmsg1(jcr, M_WARNING, 0, _("Error reading ACL of %s\n"), jcr->last_fname);
982 return true; /* no ACL */
986 if (!sd->fsend("%ld %d 0", jcr->JobFiles, stream)) {
987 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
992 /* Send the buffer to the storage deamon */
993 Dmsg2(400, "Backing up ACL type 0x%2x <%s>\n", acltype, jcr->acl_text);
995 sd->msg = jcr->acl_text;
996 sd->msglen = len + 1;
1000 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1005 jcr->JobBytes += sd->msglen;
1007 if (!sd->signal(BNET_EOD)) {
1008 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1013 Dmsg1(200, "ACL of file: %s successfully backed up!\n", jcr->last_fname);
1018 static bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream)
1020 BSOCK *sd = jcr->store_bsock;
1021 char attribs[MAXSTRING];
1022 char attribsEx[MAXSTRING];
1025 #ifdef FD_NO_SEND_TEST
1029 Dmsg1(300, "encode_and_send_attrs fname=%s\n", ff_pkt->fname);
1030 /* Find what data stream we will use, then encode the attributes */
1031 if ((data_stream = select_data_stream(ff_pkt)) == STREAM_NONE) {
1032 /* This should not happen */
1033 Jmsg0(jcr, M_FATAL, 0, _("Invalid file flags, no supported data stream type.\n"));
1036 encode_stat(attribs, ff_pkt, data_stream);
1038 /* Now possibly extend the attributes */
1039 attr_stream = encode_attribsEx(jcr, attribsEx, ff_pkt);
1041 Dmsg3(300, "File %s\nattribs=%s\nattribsEx=%s\n", ff_pkt->fname, attribs, attribsEx);
1044 jcr->JobFiles++; /* increment number of files sent */
1045 ff_pkt->FileIndex = jcr->JobFiles; /* return FileIndex */
1046 pm_strcpy(jcr->last_fname, ff_pkt->fname);
1050 * Send Attributes header to Storage daemon
1051 * <file-index> <stream> <info>
1053 if (!sd->fsend("%ld %d 0", jcr->JobFiles, attr_stream)) {
1054 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1058 Dmsg1(300, ">stored: attrhdr %s\n", sd->msg);
1061 * Send file attributes to Storage daemon
1064 * Filename (full path)
1065 * Encoded attributes
1066 * Link name (if type==FT_LNK or FT_LNKSAVED)
1067 * Encoded extended-attributes (for Win32)
1069 * For a directory, link is the same as fname, but with trailing
1070 * slash. For a linked file, link is the link.
1073 if (ff_pkt->type == FT_LNK || ff_pkt->type == FT_LNKSAVED) {
1074 Dmsg2(300, "Link %s to %s\n", ff_pkt->fname, ff_pkt->link);
1075 stat = sd->fsend("%ld %d %s%c%s%c%s%c%s%c", jcr->JobFiles,
1076 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, ff_pkt->link, 0,
1078 } else if (ff_pkt->type == FT_DIREND || ff_pkt->type == FT_REPARSE) {
1079 /* Here link is the canonical filename (i.e. with trailing slash) */
1080 stat = sd->fsend("%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
1081 ff_pkt->type, ff_pkt->link, 0, attribs, 0, 0, attribsEx, 0);
1083 stat = sd->fsend("%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
1084 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, 0, attribsEx, 0);
1086 unstrip_path(ff_pkt);
1088 Dmsg2(300, ">stored: attr len=%d: %s\n", sd->msglen, sd->msg);
1090 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1094 sd->signal(BNET_EOD); /* indicate end of attributes data */
1099 * Do in place strip of path
1101 static bool do_strip(int count, char *in)
1107 /* Copy to first path separator -- Win32 might have c: ... */
1108 while (*in && !IsPathSeparator(*in)) {
1112 numsep++; /* one separator seen */
1113 for (stripped=0; stripped<count && *in; stripped++) {
1114 while (*in && !IsPathSeparator(*in)) {
1115 in++; /* skip chars */
1118 numsep++; /* count separators seen */
1119 in++; /* skip separator */
1123 while (*in) { /* copy to end */
1124 if (IsPathSeparator(*in)) {
1130 Dmsg4(500, "stripped=%d count=%d numsep=%d sep>count=%d\n",
1131 stripped, count, numsep, numsep>count);
1132 return stripped==count && numsep>count;
1136 * If requested strip leading components of the path
1138 static void strip_path(FF_PKT *ff_pkt)
1140 if (!(ff_pkt->flags & FO_STRIPPATH) || ff_pkt->strip_path <= 0) {
1141 Dmsg1(200, "No strip for %s\n", ff_pkt->fname);
1144 if (!ff_pkt->fname_save) {
1145 ff_pkt->fname_save = get_pool_memory(PM_FNAME);
1146 ff_pkt->link_save = get_pool_memory(PM_FNAME);
1148 pm_strcpy(ff_pkt->fname_save, ff_pkt->fname);
1151 * Strip path. If it doesn't succeed put it back. If
1152 * it does, and there is a different link string,
1153 * attempt to strip the link. If it fails, but them
1155 * I.e. if either stripping fails don't strip anything.
1157 if (do_strip(ff_pkt->strip_path, ff_pkt->fname)) {
1158 if (ff_pkt->fname != ff_pkt->link) {
1159 pm_strcpy(ff_pkt->link_save, ff_pkt->link);
1160 if (!do_strip(ff_pkt->strip_path, ff_pkt->link)) {
1161 strcpy(ff_pkt->link, ff_pkt->link_save);
1162 strcpy(ff_pkt->fname, ff_pkt->fname_save);
1166 strcpy(ff_pkt->fname, ff_pkt->fname_save);
1168 Dmsg2(200, "fname=%s stripped=%s\n", ff_pkt->fname_save, ff_pkt->fname);
1171 static void unstrip_path(FF_PKT *ff_pkt)
1173 if (!(ff_pkt->flags & FO_STRIPPATH) || ff_pkt->strip_path <= 0) {
1176 strcpy(ff_pkt->fname, ff_pkt->fname_save);
1177 if (ff_pkt->fname != ff_pkt->link) {
1178 strcpy(ff_pkt->link, ff_pkt->link_save);