2 Bacula® - The Network Backup Solution
4 Copyright (C) 2000-2007 Free Software Foundation Europe e.V.
6 The main author of Bacula is Kern Sibbald, with contributions from
7 many others, a complete list can be found in the file AUTHORS.
8 This program is Free Software; you can redistribute it and/or
9 modify it under the terms of version two of the GNU General Public
10 License as published by the Free Software Foundation plus additions
11 that are listed in the file LICENSE.
13 This program is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
23 Bacula® is a registered trademark of John Walker.
24 The licensor of Bacula is the Free Software Foundation Europe
25 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
26 Switzerland, email:ftf@fsfeurope.org.
29 * Bacula File Daemon backup.c send file attributes and data
30 * to the Storage daemon.
32 * Kern Sibbald, March MM
41 /* Forward referenced functions */
42 static int save_file(FF_PKT *ff_pkt, void *pkt, bool top_level);
43 static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, DIGEST *signature_digest);
44 static bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream);
45 static bool read_and_send_acl(JCR *jcr, int acltype, int stream);
48 * Find all the requested files and send them
49 * to the Storage daemon.
51 * Note, we normally carry on a one-way
52 * conversation from this point on with the SD, simply blasting
53 * data to him. To properly know what is going on, we
54 * also run a "heartbeat" monitor which reads the socket and
55 * reacts accordingly (at the moment it has nothing to do
56 * except echo the heartbeat to the Director).
59 bool blast_data_to_storage_daemon(JCR *jcr, char *addr)
63 // TODO landonf: Allow user to specify encryption algorithm
64 crypto_cipher_t cipher = CRYPTO_CIPHER_AES_128_CBC;
67 sd = jcr->store_bsock;
69 set_jcr_job_status(jcr, JS_Running);
71 Dmsg1(300, "bfiled: opened data connection %d to stored\n", sd->fd);
74 CLIENT *client = (CLIENT *)GetNextRes(R_CLIENT, NULL);
78 buf_size = client->max_network_buffer_size;
80 buf_size = 0; /* use default */
82 if (!bnet_set_buffer_size(sd, buf_size, BNET_SETBUF_WRITE)) {
83 set_jcr_job_status(jcr, JS_ErrorTerminated);
84 Jmsg(jcr, M_FATAL, 0, _("Cannot set buffer size FD->SD.\n"));
88 jcr->buf_size = sd->msglen;
89 /* Adjust for compression so that output buffer is
90 * 12 bytes + 0.1% larger than input buffer plus 18 bytes.
91 * This gives a bit extra plus room for the sparse addr if any.
92 * Note, we adjust the read size to be smaller so that the
93 * same output buffer can be used without growing it.
95 * The zlib compression workset is initialized here to minimise
96 * the "per file" load. The jcr member is only set, if the init was successful.
98 jcr->compress_buf_size = jcr->buf_size + ((jcr->buf_size+999) / 1000) + 30;
99 jcr->compress_buf = get_memory(jcr->compress_buf_size);
102 z_stream *pZlibStream = (z_stream*)malloc(sizeof(z_stream));
104 pZlibStream->zalloc = Z_NULL;
105 pZlibStream->zfree = Z_NULL;
106 pZlibStream->opaque = Z_NULL;
107 pZlibStream->state = Z_NULL;
109 if (deflateInit(pZlibStream, Z_DEFAULT_COMPRESSION) == Z_OK)
110 jcr->pZLIB_compress_workset = pZlibStream;
116 /* Create encryption session data and a cached, DER-encoded session data
117 * structure. We use a single session key for each backup, so we'll encode
118 * the session data only once. */
119 if (jcr->pki_encrypt) {
122 /* Create per-job session encryption context */
123 jcr->pki_session = crypto_session_new(cipher, jcr->pki_recipients);
125 /* Get the session data size */
126 if (crypto_session_encode(jcr->pki_session, (uint8_t *)0, &size) == false) {
127 Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n"));
131 /* Allocate buffer */
132 jcr->pki_session_encoded = (uint8_t *)malloc(size);
133 if (!jcr->pki_session_encoded) {
137 /* Encode session data */
138 if (crypto_session_encode(jcr->pki_session, jcr->pki_session_encoded, &size) == false) {
139 Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n"));
143 /* ... and store the encoded size */
144 jcr->pki_session_encoded_size = size;
146 /* Allocate the encryption/decryption buffer */
147 jcr->crypto_buf = get_memory(CRYPTO_CIPHER_MAX_BLOCK_SIZE);
150 Dmsg1(300, "set_find_options ff=%p\n", jcr->ff);
151 set_find_options((FF_PKT *)jcr->ff, jcr->incremental, jcr->mtime);
152 Dmsg0(300, "start find files\n");
154 start_heartbeat_monitor(jcr);
156 jcr->acl_text = get_pool_memory(PM_MESSAGE);
158 /* Subroutine save_file() is called for each file */
159 if (!find_files(jcr, (FF_PKT *)jcr->ff, save_file, (void *)jcr)) {
160 ok = false; /* error */
161 set_jcr_job_status(jcr, JS_ErrorTerminated);
164 free_pool_memory(jcr->acl_text);
166 stop_heartbeat_monitor(jcr);
168 bnet_sig(sd, BNET_EOD); /* end of sending data */
174 if (jcr->compress_buf) {
175 free_pool_memory(jcr->compress_buf);
176 jcr->compress_buf = NULL;
178 if (jcr->pZLIB_compress_workset) {
179 /* Free the zlib stream */
181 deflateEnd((z_stream *)jcr->pZLIB_compress_workset);
183 free (jcr->pZLIB_compress_workset);
184 jcr->pZLIB_compress_workset = NULL;
186 if (jcr->crypto_buf) {
187 free_pool_memory(jcr->crypto_buf);
188 jcr->crypto_buf = NULL;
190 if (jcr->pki_session) {
191 crypto_session_free(jcr->pki_session);
193 if (jcr->pki_session_encoded) {
194 free(jcr->pki_session_encoded);
197 Dmsg1(100, "end blast_data ok=%d\n", ok);
202 * Called here by find() for each file included.
203 * This is a callback. The original is find_files() above.
205 * Send the file and its data to the Storage daemon.
209 * -1 to ignore file/directory (not used here)
211 static int save_file(FF_PKT *ff_pkt, void *vjcr, bool top_level)
213 bool do_read = false;
214 int stat, data_stream;
216 DIGEST *digest = NULL;
217 DIGEST *signing_digest = NULL;
218 int digest_stream = STREAM_NONE;
219 SIGNATURE *sig = NULL;
221 bool has_file_data = false;
222 // TODO landonf: Allow the user to specify the digest algorithm
224 crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA256;
226 crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA1;
228 JCR *jcr = (JCR *)vjcr;
229 BSOCK *sd = jcr->store_bsock;
231 if (job_canceled(jcr)) {
235 jcr->num_files_examined++; /* bump total file count */
237 switch (ff_pkt->type) {
238 case FT_LNKSAVED: /* Hard linked, file already saved */
239 Dmsg2(130, "FT_LNKSAVED hard link: %s => %s\n", ff_pkt->fname, ff_pkt->link);
242 Dmsg1(130, "FT_REGE saving: %s\n", ff_pkt->fname);
243 has_file_data = true;
246 Dmsg1(130, "FT_REG saving: %s\n", ff_pkt->fname);
247 has_file_data = true;
250 Dmsg2(130, "FT_LNK saving: %s -> %s\n", ff_pkt->fname, ff_pkt->link);
253 jcr->num_files_examined--; /* correct file count */
254 return 1; /* not used */
256 Jmsg(jcr, M_INFO, 1, _(" Recursion turned off. Will not descend from %s into %s\n"),
257 ff_pkt->top_fname, ff_pkt->fname);
258 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
261 /* Suppress message for /dev filesystems */
262 if (strncmp(ff_pkt->fname, "/dev/", 5) != 0) {
263 Jmsg(jcr, M_INFO, 1, _(" %s is a different filesystem. Will not descend from %s into %s\n"),
264 ff_pkt->fname, ff_pkt->top_fname, ff_pkt->fname);
266 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
269 Jmsg(jcr, M_INFO, 1, _(" Disallowed filesystem. Will not descend from %s into %s\n"),
270 ff_pkt->top_fname, ff_pkt->fname);
271 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
274 Jmsg(jcr, M_INFO, 1, _(" Disallowed drive type. Will not descend into %s\n"),
278 Dmsg1(130, "FT_DIREND: %s\n", ff_pkt->link);
281 Dmsg1(130, "FT_SPEC saving: %s\n", ff_pkt->fname);
284 Dmsg1(130, "FT_RAW saving: %s\n", ff_pkt->fname);
285 has_file_data = true;
288 Dmsg1(130, "FT_FIFO saving: %s\n", ff_pkt->fname);
292 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not access %s: ERR=%s\n"), ff_pkt->fname,
293 be.strerror(ff_pkt->ff_errno));
299 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not follow link %s: ERR=%s\n"), ff_pkt->fname,
300 be.strerror(ff_pkt->ff_errno));
306 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not stat %s: ERR=%s\n"), ff_pkt->fname,
307 be.strerror(ff_pkt->ff_errno));
313 Jmsg(jcr, M_SKIPPED, 1, _(" Unchanged file skipped: %s\n"), ff_pkt->fname);
316 Jmsg(jcr, M_NOTSAVED, 0, _(" Archive file not saved: %s\n"), ff_pkt->fname);
320 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not open directory %s: ERR=%s\n"), ff_pkt->fname,
321 be.strerror(ff_pkt->ff_errno));
326 Jmsg(jcr, M_NOTSAVED, 0, _(" Unknown file type %d; not saved: %s\n"), ff_pkt->type, ff_pkt->fname);
331 Dmsg1(130, "bfiled: sending %s to stored\n", ff_pkt->fname);
333 /* Digests and encryption are only useful if there's file data */
336 * Setup for digest handling. If this fails, the digest will be set to NULL
339 if (ff_pkt->flags & FO_MD5) {
340 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_MD5);
341 digest_stream = STREAM_MD5_DIGEST;
343 } else if (ff_pkt->flags & FO_SHA1) {
344 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA1);
345 digest_stream = STREAM_SHA1_DIGEST;
347 } else if (ff_pkt->flags & FO_SHA256) {
348 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA256);
349 digest_stream = STREAM_SHA256_DIGEST;
351 } else if (ff_pkt->flags & FO_SHA512) {
352 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA512);
353 digest_stream = STREAM_SHA512_DIGEST;
356 /* Did digest initialization fail? */
357 if (digest_stream != STREAM_NONE && digest == NULL) {
358 Jmsg(jcr, M_WARNING, 0, _("%s digest initialization failed\n"),
359 stream_to_ascii(digest_stream));
363 * Set up signature digest handling. If this fails, the signature digest will be set to
366 // TODO landonf: We should really only calculate the digest once, for both verification and signing.
368 signing_digest = crypto_digest_new(jcr, signing_algorithm);
370 /* Full-stop if a failure occurred initializing the signature digest */
371 if (signing_digest == NULL) {
372 Jmsg(jcr, M_NOTSAVED, 0, _("%s signature digest initialization failed\n"),
373 stream_to_ascii(signing_algorithm));
379 /* Enable encryption */
380 if (jcr->pki_encrypt) {
381 ff_pkt->flags |= FO_ENCRYPT;
385 /* Initialise the file descriptor we use for data and other streams. */
387 if (ff_pkt->flags & FO_PORTABLE) {
388 set_portable_backup(&ff_pkt->bfd); /* disable Win32 BackupRead() */
390 if (ff_pkt->reader) {
391 if (!set_prog(&ff_pkt->bfd, ff_pkt->reader, jcr)) {
392 Jmsg(jcr, M_FATAL, 0, _("Python reader program \"%s\" not found.\n"),
398 /* Send attributes -- must be done after binit() */
399 if (!encode_and_send_attributes(jcr, ff_pkt, data_stream)) {
403 /* Set up the encryption context and send the session data to the SD */
404 if (has_file_data && jcr->pki_encrypt) {
405 /* Send our header */
406 Dmsg2(100, "Send hdr fi=%ld stream=%d\n", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA);
407 bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA);
409 /* Grow the bsock buffer to fit our message if necessary */
410 if (sizeof_pool_memory(sd->msg) < jcr->pki_session_encoded_size) {
411 sd->msg = realloc_pool_memory(sd->msg, jcr->pki_session_encoded_size);
414 /* Copy our message over and send it */
415 memcpy(sd->msg, jcr->pki_session_encoded, jcr->pki_session_encoded_size);
416 sd->msglen = jcr->pki_session_encoded_size;
417 jcr->JobBytes += sd->msglen;
419 Dmsg1(100, "Send data len=%d\n", sd->msglen);
421 bnet_sig(sd, BNET_EOD);
425 * Open any file with data that we intend to save, then save it.
427 * Note, if is_win32_backup, we must open the Directory so that
428 * the BackupRead will save its permissions and ownership streams.
430 if (ff_pkt->type != FT_LNKSAVED && S_ISREG(ff_pkt->statp.st_mode)) {
432 do_read = !is_portable_backup(&ff_pkt->bfd) || ff_pkt->statp.st_size > 0;
434 do_read = ff_pkt->statp.st_size > 0;
436 } else if (ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO ||
437 (!is_portable_backup(&ff_pkt->bfd) && ff_pkt->type == FT_DIREND)) {
443 if (ff_pkt->type == FT_FIFO) {
444 tid = start_thread_timer(pthread_self(), 60);
448 int noatime = ff_pkt->flags & FO_NOATIME ? O_NOATIME : 0;
449 if (bopen(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY | noatime, 0) < 0) {
450 ff_pkt->ff_errno = errno;
452 Jmsg(jcr, M_NOTSAVED, 0, _(" Cannot open %s: ERR=%s.\n"), ff_pkt->fname,
456 stop_thread_timer(tid);
462 stop_thread_timer(tid);
466 stat = send_data(jcr, data_stream, ff_pkt, digest, signing_digest);
468 if (ff_pkt->flags & FO_CHKCHANGES) {
469 has_file_changed(jcr, ff_pkt);
472 bclose(&ff_pkt->bfd);
479 #ifdef HAVE_DARWIN_OS
480 /* Regular files can have resource forks and Finder Info */
481 if (ff_pkt->type != FT_LNKSAVED && (S_ISREG(ff_pkt->statp.st_mode) &&
482 ff_pkt->flags & FO_HFSPLUS)) {
483 if (ff_pkt->hfsinfo.rsrclength > 0) {
486 if (!bopen_rsrc(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0) < 0) {
487 ff_pkt->ff_errno = errno;
489 Jmsg(jcr, M_NOTSAVED, -1, _(" Cannot open resource fork for %s: ERR=%s.\n"), ff_pkt->fname,
492 if (is_bopen(&ff_pkt->bfd)) {
493 bclose(&ff_pkt->bfd);
497 flags = ff_pkt->flags;
498 ff_pkt->flags &= ~(FO_GZIP|FO_SPARSE);
499 if (flags & FO_ENCRYPT) {
500 rsrc_stream = STREAM_ENCRYPTED_MACOS_FORK_DATA;
502 rsrc_stream = STREAM_MACOS_FORK_DATA;
504 stat = send_data(jcr, rsrc_stream, ff_pkt, digest, signing_digest);
505 ff_pkt->flags = flags;
506 bclose(&ff_pkt->bfd);
512 Dmsg1(300, "Saving Finder Info for \"%s\"\n", ff_pkt->fname);
513 bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, STREAM_HFSPLUS_ATTRIBUTES);
514 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
515 memcpy(sd->msg, ff_pkt->hfsinfo.fndrinfo, 32);
518 crypto_digest_update(digest, (uint8_t *)sd->msg, sd->msglen);
520 if (signing_digest) {
521 crypto_digest_update(signing_digest, (uint8_t *)sd->msg, sd->msglen);
524 bnet_sig(sd, BNET_EOD);
528 if (ff_pkt->flags & FO_ACL) {
529 /* Read access ACLs for files, dirs and links */
530 if (!read_and_send_acl(jcr, BACL_TYPE_ACCESS, STREAM_UNIX_ATTRIBUTES_ACCESS_ACL)) {
533 /* Directories can have default ACLs too */
534 if (ff_pkt->type == FT_DIREND && (BACL_CAP & BACL_CAP_DEFAULTS_DIR)) {
535 if (!read_and_send_acl(jcr, BACL_TYPE_DEFAULT, STREAM_UNIX_ATTRIBUTES_DEFAULT_ACL)) {
541 /* Terminate the signing digest and send it to the Storage daemon */
542 if (signing_digest) {
545 if ((sig = crypto_sign_new(jcr)) == NULL) {
546 Jmsg(jcr, M_FATAL, 0, _("Failed to allocate memory for crypto signature.\n"));
550 if (crypto_sign_add_signer(sig, signing_digest, jcr->pki_keypair) == false) {
551 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
555 /* Get signature size */
556 if (crypto_sign_encode(sig, NULL, &size) == false) {
557 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
561 /* Allocate signature data buffer */
562 buf = (uint8_t *)malloc(size);
567 /* Encode signature data */
568 if (crypto_sign_encode(sig, buf, &size) == false) {
569 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
573 /* Send our header */
574 bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, STREAM_SIGNED_DIGEST);
575 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
577 /* Grow the bsock buffer to fit our message if necessary */
578 if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
579 sd->msg = realloc_pool_memory(sd->msg, size);
582 /* Copy our message over and send it */
583 memcpy(sd->msg, buf, size);
586 bnet_sig(sd, BNET_EOD); /* end of checksum */
589 /* Terminate any digest and send it to Storage daemon and the Director */
591 uint8_t md[CRYPTO_DIGEST_MAX_SIZE];
596 if (crypto_digest_finalize(digest, md, &size)) {
597 bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, digest_stream);
598 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
599 memcpy(sd->msg, md, size);
602 bnet_sig(sd, BNET_EOD); /* end of checksum */
607 rtnstat = 1; /* good return */
611 crypto_digest_free(digest);
613 if (signing_digest) {
614 crypto_digest_free(signing_digest);
617 crypto_sign_free(sig);
626 * Send data read from an already open file descriptor.
628 * We return 1 on sucess and 0 on errors.
631 * We use ff_pkt->statp.st_size when FO_SPARSE to know when to stop
633 * Currently this is not a problem as the only other stream, resource forks,
634 * are not handled as sparse files.
636 int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest,
637 DIGEST *signing_digest)
639 BSOCK *sd = jcr->store_bsock;
640 uint64_t fileAddr = 0; /* file address */
642 int32_t rsize = jcr->buf_size; /* read buffer size */
644 CIPHER_CONTEXT *cipher_ctx = NULL; /* Quell bogus uninitialized warnings */
645 const uint8_t *cipher_input;
646 uint32_t cipher_input_len;
647 uint32_t cipher_block_size;
648 uint32_t encrypted_len;
649 #ifdef FD_NO_SEND_TEST
654 rbuf = sd->msg; /* read buffer */
655 wbuf = sd->msg; /* write buffer */
656 cipher_input = (uint8_t *)rbuf; /* encrypt uncompressed data */
658 Dmsg1(300, "Saving data, type=%d\n", ff_pkt->type);
661 uLong compress_len = 0;
662 uLong max_compress_len = 0;
663 const Bytef *cbuf = NULL;
666 if (ff_pkt->flags & FO_GZIP) {
667 if (ff_pkt->flags & FO_SPARSE) {
668 cbuf = (Bytef *)jcr->compress_buf + SPARSE_FADDR_SIZE;
669 max_compress_len = jcr->compress_buf_size - SPARSE_FADDR_SIZE;
671 cbuf = (Bytef *)jcr->compress_buf;
672 max_compress_len = jcr->compress_buf_size; /* set max length */
674 wbuf = jcr->compress_buf; /* compressed output here */
675 cipher_input = (uint8_t *)jcr->compress_buf; /* encrypt compressed data */
678 * Only change zlib parameters if there is no pending operation.
679 * This should never happen as deflatereset is called after each
683 if (((z_stream*)jcr->pZLIB_compress_workset)->total_in == 0) {
684 /* set gzip compression level - must be done per file */
685 if ((zstat=deflateParams((z_stream*)jcr->pZLIB_compress_workset,
686 ff_pkt->GZIP_level, Z_DEFAULT_STRATEGY)) != Z_OK) {
687 Jmsg(jcr, M_FATAL, 0, _("Compression deflateParams error: %d\n"), zstat);
688 set_jcr_job_status(jcr, JS_ErrorTerminated);
694 const uint32_t max_compress_len = 0;
697 if (ff_pkt->flags & FO_ENCRYPT) {
698 if (ff_pkt->flags & FO_SPARSE) {
699 Jmsg0(jcr, M_FATAL, 0, _("Encrypting sparse data not supported.\n"));
702 /* Allocate the cipher context */
703 if ((cipher_ctx = crypto_cipher_new(jcr->pki_session, true,
704 &cipher_block_size)) == NULL) {
705 /* Shouldn't happen! */
706 Jmsg0(jcr, M_FATAL, 0, _("Failed to initialize encryption context.\n"));
711 * Grow the crypto buffer, if necessary.
712 * crypto_cipher_update() will buffer up to (cipher_block_size - 1).
713 * We grow crypto_buf to the maximum number of blocks that
714 * could be returned for the given read buffer size.
715 * (Using the larger of either rsize or max_compress_len)
717 jcr->crypto_buf = check_pool_memory_size(jcr->crypto_buf,
718 (MAX(rsize + (int)sizeof(uint32_t), (int32_t)max_compress_len) +
719 cipher_block_size - 1) / cipher_block_size * cipher_block_size);
721 wbuf = jcr->crypto_buf; /* Encrypted, possibly compressed output here. */
725 * Send Data header to Storage daemon
726 * <file-index> <stream> <info>
728 if (!bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, stream)) {
729 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
733 Dmsg1(300, ">stored: datahdr %s\n", sd->msg);
736 * Make space at beginning of buffer for fileAddr because this
737 * same buffer will be used for writing if compression is off.
739 if (ff_pkt->flags & FO_SPARSE) {
740 rbuf += SPARSE_FADDR_SIZE;
741 rsize -= SPARSE_FADDR_SIZE;
742 #ifdef HAVE_FREEBSD_OS
744 * To read FreeBSD partitions, the read size must be
747 rsize = (rsize/512) * 512;
751 /* a RAW device read on win32 only works if the buffer is a multiple of 512 */
753 if (S_ISBLK(ff_pkt->statp.st_mode))
754 rsize = (rsize/512) * 512;
760 while ((sd->msglen=(uint32_t)bread(&ff_pkt->bfd, rbuf, rsize)) > 0) {
762 /* Check for sparse blocks */
763 if (ff_pkt->flags & FO_SPARSE) {
765 bool haveBlock = true;
766 if (sd->msglen == rsize &&
767 fileAddr+sd->msglen < (uint64_t)ff_pkt->statp.st_size ||
768 ((ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO) &&
769 (uint64_t)ff_pkt->statp.st_size == 0)) {
770 haveBlock = !is_buf_zero(rbuf, rsize);
773 ser_begin(wbuf, SPARSE_FADDR_SIZE);
774 ser_uint64(fileAddr); /* store fileAddr in begin of buffer */
776 fileAddr += sd->msglen; /* update file address */
778 continue; /* skip block of zeros */
782 jcr->ReadBytes += sd->msglen; /* count bytes read */
784 /* Uncompressed cipher input length */
785 cipher_input_len = sd->msglen;
787 /* Update checksum if requested */
789 crypto_digest_update(digest, (uint8_t *)rbuf, sd->msglen);
792 /* Update signing digest if requested */
793 if (signing_digest) {
794 crypto_digest_update(signing_digest, (uint8_t *)rbuf, sd->msglen);
798 /* Do compression if turned on */
799 if (ff_pkt->flags & FO_GZIP && jcr->pZLIB_compress_workset) {
800 Dmsg3(400, "cbuf=0x%x rbuf=0x%x len=%u\n", cbuf, rbuf, sd->msglen);
802 ((z_stream*)jcr->pZLIB_compress_workset)->next_in = (Bytef *)rbuf;
803 ((z_stream*)jcr->pZLIB_compress_workset)->avail_in = sd->msglen;
804 ((z_stream*)jcr->pZLIB_compress_workset)->next_out = (Bytef *)cbuf;
805 ((z_stream*)jcr->pZLIB_compress_workset)->avail_out = max_compress_len;
807 if ((zstat=deflate((z_stream*)jcr->pZLIB_compress_workset, Z_FINISH)) != Z_STREAM_END) {
808 Jmsg(jcr, M_FATAL, 0, _("Compression deflate error: %d\n"), zstat);
809 set_jcr_job_status(jcr, JS_ErrorTerminated);
812 compress_len = ((z_stream*)jcr->pZLIB_compress_workset)->total_out;
813 /* reset zlib stream to be able to begin from scratch again */
814 if ((zstat=deflateReset((z_stream*)jcr->pZLIB_compress_workset)) != Z_OK) {
815 Jmsg(jcr, M_FATAL, 0, _("Compression deflateReset error: %d\n"), zstat);
816 set_jcr_job_status(jcr, JS_ErrorTerminated);
820 Dmsg2(400, "compressed len=%d uncompressed len=%d\n", compress_len,
823 sd->msglen = compress_len; /* set compressed length */
824 cipher_input_len = compress_len;
828 * Note, here we prepend the current record length to the beginning
829 * of the encrypted data. This is because both sparse and compression
830 * restore handling want records returned to them with exactly the
831 * same number of bytes that were processed in the backup handling.
832 * That is, both are block filters rather than a stream. When doing
833 * compression, the compression routines may buffer data, so that for
834 * any one record compressed, when it is decompressed the same size
835 * will not be obtained. Of course, the buffered data eventually comes
836 * out in subsequent crypto_cipher_update() calls or at least
837 * when crypto_cipher_finalize() is called. Unfortunately, this
838 * "feature" of encryption enormously complicates the restore code.
840 if (ff_pkt->flags & FO_ENCRYPT) {
841 uint32_t initial_len = 0;
844 if (ff_pkt->flags & FO_SPARSE) {
845 cipher_input_len += SPARSE_FADDR_SIZE;
848 /* Encrypt the length of the input block */
849 uint8_t packet_len[sizeof(uint32_t)];
851 ser_begin(packet_len, sizeof(uint32_t));
852 ser_uint32(cipher_input_len); /* store data len in begin of buffer */
853 Dmsg1(20, "Encrypt len=%d\n", cipher_input_len);
855 if (!crypto_cipher_update(cipher_ctx, packet_len, sizeof(packet_len),
856 (u_int8_t *)jcr->crypto_buf, &initial_len)) {
857 /* Encryption failed. Shouldn't happen. */
858 Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
862 /* Encrypt the input block */
863 if (crypto_cipher_update(cipher_ctx, cipher_input, cipher_input_len,
864 (u_int8_t *)&jcr->crypto_buf[initial_len], &encrypted_len)) {
865 if ((initial_len + encrypted_len) == 0) {
866 /* No full block of data available, read more data */
869 Dmsg2(400, "encrypted len=%d unencrypted len=%d\n", encrypted_len,
871 sd->msglen = initial_len + encrypted_len; /* set encrypted length */
873 /* Encryption failed. Shouldn't happen. */
874 Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
879 /* Send the buffer to the Storage daemon */
880 if (ff_pkt->flags & FO_SPARSE) {
881 sd->msglen += SPARSE_FADDR_SIZE; /* include fileAddr in size */
883 sd->msg = wbuf; /* set correct write buffer */
884 if (!bnet_send(sd)) {
885 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
889 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
891 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
892 sd->msg = msgsave; /* restore read buffer */
894 } /* end while read file data */
896 if (sd->msglen < 0) { /* error */
898 Jmsg(jcr, M_ERROR, 0, _("Read error on file %s. ERR=%s\n"),
899 ff_pkt->fname, be.strerror(ff_pkt->bfd.berrno));
900 if (jcr->Errors++ > 1000) { /* insanity check */
901 Jmsg(jcr, M_FATAL, 0, _("Too many errors.\n"));
903 } else if (ff_pkt->flags & FO_ENCRYPT) {
905 * For encryption, we must call finalize to push out any
908 if (!crypto_cipher_finalize(cipher_ctx, (uint8_t *)jcr->crypto_buf,
910 /* Padding failed. Shouldn't happen. */
911 Jmsg(jcr, M_FATAL, 0, _("Encryption padding error\n"));
915 /* Note, on SSL pre-0.9.7, there is always some output */
916 if (encrypted_len > 0) {
917 sd->msglen = encrypted_len; /* set encrypted length */
918 sd->msg = jcr->crypto_buf; /* set correct write buffer */
919 if (!bnet_send(sd)) {
920 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
924 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
925 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
926 sd->msg = msgsave; /* restore bnet buffer */
930 if (!bnet_sig(sd, BNET_EOD)) { /* indicate end of file data */
931 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
936 /* Free the cipher context */
938 crypto_cipher_free(cipher_ctx);
943 /* Free the cipher context */
945 crypto_cipher_free(cipher_ctx);
948 sd->msg = msgsave; /* restore bnet buffer */
954 * Read and send an ACL for the last encountered file.
956 static bool read_and_send_acl(JCR *jcr, int acltype, int stream)
959 BSOCK *sd = jcr->store_bsock;
962 #ifdef FD_NO_SEND_TEST
966 len = bacl_get(jcr, acltype);
968 Jmsg1(jcr, M_WARNING, 0, _("Error reading ACL of %s\n"), jcr->last_fname);
972 return true; /* no ACL */
976 if (!bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, stream)) {
977 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
982 /* Send the buffer to the storage deamon */
983 Dmsg2(400, "Backing up ACL type 0x%2x <%s>\n", acltype, jcr->acl_text);
985 sd->msg = jcr->acl_text;
986 sd->msglen = len + 1;
987 if (!bnet_send(sd)) {
990 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
995 jcr->JobBytes += sd->msglen;
997 if (!bnet_sig(sd, BNET_EOD)) {
998 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1003 Dmsg1(200, "ACL of file: %s successfully backed up!\n", jcr->last_fname);
1008 static bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream)
1010 BSOCK *sd = jcr->store_bsock;
1011 char attribs[MAXSTRING];
1012 char attribsEx[MAXSTRING];
1015 #ifdef FD_NO_SEND_TEST
1019 /* Find what data stream we will use, then encode the attributes */
1020 if ((data_stream = select_data_stream(ff_pkt)) == STREAM_NONE) {
1021 /* This should not happen */
1022 Jmsg0(jcr, M_FATAL, 0, _("Invalid file flags, no supported data stream type.\n"));
1025 encode_stat(attribs, ff_pkt, data_stream);
1027 /* Now possibly extend the attributes */
1028 attr_stream = encode_attribsEx(jcr, attribsEx, ff_pkt);
1030 Dmsg3(300, "File %s\nattribs=%s\nattribsEx=%s\n", ff_pkt->fname, attribs, attribsEx);
1033 jcr->JobFiles++; /* increment number of files sent */
1034 ff_pkt->FileIndex = jcr->JobFiles; /* return FileIndex */
1035 pm_strcpy(jcr->last_fname, ff_pkt->fname);
1039 * Send Attributes header to Storage daemon
1040 * <file-index> <stream> <info>
1042 if (!bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, attr_stream)) {
1043 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1047 Dmsg1(300, ">stored: attrhdr %s\n", sd->msg);
1050 * Send file attributes to Storage daemon
1053 * Filename (full path)
1054 * Encoded attributes
1055 * Link name (if type==FT_LNK or FT_LNKSAVED)
1056 * Encoded extended-attributes (for Win32)
1058 * For a directory, link is the same as fname, but with trailing
1059 * slash. For a linked file, link is the link.
1061 if (ff_pkt->type == FT_LNK || ff_pkt->type == FT_LNKSAVED) {
1062 Dmsg2(300, "Link %s to %s\n", ff_pkt->fname, ff_pkt->link);
1063 stat = bnet_fsend(sd, "%ld %d %s%c%s%c%s%c%s%c", jcr->JobFiles,
1064 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, ff_pkt->link, 0,
1066 } else if (ff_pkt->type == FT_DIREND) {
1067 /* Here link is the canonical filename (i.e. with trailing slash) */
1068 stat = bnet_fsend(sd, "%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
1069 ff_pkt->type, ff_pkt->link, 0, attribs, 0, 0, attribsEx, 0);
1071 stat = bnet_fsend(sd, "%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
1072 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, 0, attribsEx, 0);
1075 Dmsg2(300, ">stored: attr len=%d: %s\n", sd->msglen, sd->msg);
1077 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1081 bnet_sig(sd, BNET_EOD); /* indicate end of attributes data */