2 Bacula® - The Network Backup Solution
4 Copyright (C) 2000-2007 Free Software Foundation Europe e.V.
6 The main author of Bacula is Kern Sibbald, with contributions from
7 many others, a complete list can be found in the file AUTHORS.
8 This program is Free Software; you can redistribute it and/or
9 modify it under the terms of version two of the GNU General Public
10 License as published by the Free Software Foundation and included
13 This program is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
23 Bacula® is a registered trademark of John Walker.
24 The licensor of Bacula is the Free Software Foundation Europe
25 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
26 Switzerland, email:ftf@fsfeurope.org.
29 * Bacula File Daemon backup.c send file attributes and data
30 * to the Storage daemon.
32 * Kern Sibbald, March MM
41 /* Forward referenced functions */
42 static int save_file(FF_PKT *ff_pkt, void *pkt, bool top_level);
43 static void strip_path(FF_PKT *ff_pkt);
44 static void unstrip_path(FF_PKT *ff_pkt);
45 static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, DIGEST *signature_digest);
46 static bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream);
47 static bool read_and_send_acl(JCR *jcr, int acltype, int stream);
50 * Find all the requested files and send them
51 * to the Storage daemon.
53 * Note, we normally carry on a one-way
54 * conversation from this point on with the SD, simply blasting
55 * data to him. To properly know what is going on, we
56 * also run a "heartbeat" monitor which reads the socket and
57 * reacts accordingly (at the moment it has nothing to do
58 * except echo the heartbeat to the Director).
61 bool blast_data_to_storage_daemon(JCR *jcr, char *addr)
65 // TODO landonf: Allow user to specify encryption algorithm
66 crypto_cipher_t cipher = CRYPTO_CIPHER_AES_128_CBC;
69 sd = jcr->store_bsock;
71 set_jcr_job_status(jcr, JS_Running);
73 Dmsg1(300, "bfiled: opened data connection %d to stored\n", sd->m_fd);
76 CLIENT *client = (CLIENT *)GetNextRes(R_CLIENT, NULL);
80 buf_size = client->max_network_buffer_size;
82 buf_size = 0; /* use default */
84 if (!bnet_set_buffer_size(sd, buf_size, BNET_SETBUF_WRITE)) {
85 set_jcr_job_status(jcr, JS_ErrorTerminated);
86 Jmsg(jcr, M_FATAL, 0, _("Cannot set buffer size FD->SD.\n"));
90 jcr->buf_size = sd->msglen;
91 /* Adjust for compression so that output buffer is
92 * 12 bytes + 0.1% larger than input buffer plus 18 bytes.
93 * This gives a bit extra plus room for the sparse addr if any.
94 * Note, we adjust the read size to be smaller so that the
95 * same output buffer can be used without growing it.
97 * The zlib compression workset is initialized here to minimise
98 * the "per file" load. The jcr member is only set, if the init was successful.
100 jcr->compress_buf_size = jcr->buf_size + ((jcr->buf_size+999) / 1000) + 30;
101 jcr->compress_buf = get_memory(jcr->compress_buf_size);
104 z_stream *pZlibStream = (z_stream*)malloc(sizeof(z_stream));
106 pZlibStream->zalloc = Z_NULL;
107 pZlibStream->zfree = Z_NULL;
108 pZlibStream->opaque = Z_NULL;
109 pZlibStream->state = Z_NULL;
111 if (deflateInit(pZlibStream, Z_DEFAULT_COMPRESSION) == Z_OK)
112 jcr->pZLIB_compress_workset = pZlibStream;
118 /* Create encryption session data and a cached, DER-encoded session data
119 * structure. We use a single session key for each backup, so we'll encode
120 * the session data only once. */
121 if (jcr->pki_encrypt) {
124 /* Create per-job session encryption context */
125 jcr->pki_session = crypto_session_new(cipher, jcr->pki_recipients);
127 /* Get the session data size */
128 if (crypto_session_encode(jcr->pki_session, (uint8_t *)0, &size) == false) {
129 Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n"));
133 /* Allocate buffer */
134 jcr->pki_session_encoded = (uint8_t *)malloc(size);
135 if (!jcr->pki_session_encoded) {
139 /* Encode session data */
140 if (crypto_session_encode(jcr->pki_session, jcr->pki_session_encoded, &size) == false) {
141 Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n"));
145 /* ... and store the encoded size */
146 jcr->pki_session_encoded_size = size;
148 /* Allocate the encryption/decryption buffer */
149 jcr->crypto_buf = get_memory(CRYPTO_CIPHER_MAX_BLOCK_SIZE);
152 Dmsg1(300, "set_find_options ff=%p\n", jcr->ff);
153 set_find_options((FF_PKT *)jcr->ff, jcr->incremental, jcr->mtime);
154 Dmsg0(300, "start find files\n");
156 start_heartbeat_monitor(jcr);
158 jcr->acl_text = get_pool_memory(PM_MESSAGE);
160 /* Subroutine save_file() is called for each file */
161 if (!find_files(jcr, (FF_PKT *)jcr->ff, save_file, (void *)jcr)) {
162 ok = false; /* error */
163 set_jcr_job_status(jcr, JS_ErrorTerminated);
166 free_pool_memory(jcr->acl_text);
168 stop_heartbeat_monitor(jcr);
170 bnet_sig(sd, BNET_EOD); /* end of sending data */
176 if (jcr->compress_buf) {
177 free_pool_memory(jcr->compress_buf);
178 jcr->compress_buf = NULL;
180 if (jcr->pZLIB_compress_workset) {
181 /* Free the zlib stream */
183 deflateEnd((z_stream *)jcr->pZLIB_compress_workset);
185 free (jcr->pZLIB_compress_workset);
186 jcr->pZLIB_compress_workset = NULL;
188 if (jcr->crypto_buf) {
189 free_pool_memory(jcr->crypto_buf);
190 jcr->crypto_buf = NULL;
192 if (jcr->pki_session) {
193 crypto_session_free(jcr->pki_session);
195 if (jcr->pki_session_encoded) {
196 free(jcr->pki_session_encoded);
199 Dmsg1(100, "end blast_data ok=%d\n", ok);
204 * Called here by find() for each file included.
205 * This is a callback. The original is find_files() above.
207 * Send the file and its data to the Storage daemon.
211 * -1 to ignore file/directory (not used here)
213 static int save_file(FF_PKT *ff_pkt, void *vjcr, bool top_level)
215 bool do_read = false;
216 int stat, data_stream;
218 DIGEST *digest = NULL;
219 DIGEST *signing_digest = NULL;
220 int digest_stream = STREAM_NONE;
221 SIGNATURE *sig = NULL;
223 bool has_file_data = false;
224 // TODO landonf: Allow the user to specify the digest algorithm
226 crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA256;
228 crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA1;
230 JCR *jcr = (JCR *)vjcr;
231 BSOCK *sd = jcr->store_bsock;
233 if (job_canceled(jcr)) {
237 jcr->num_files_examined++; /* bump total file count */
239 switch (ff_pkt->type) {
240 case FT_LNKSAVED: /* Hard linked, file already saved */
241 Dmsg2(130, "FT_LNKSAVED hard link: %s => %s\n", ff_pkt->fname, ff_pkt->link);
244 Dmsg1(130, "FT_REGE saving: %s\n", ff_pkt->fname);
245 has_file_data = true;
248 Dmsg1(130, "FT_REG saving: %s\n", ff_pkt->fname);
249 has_file_data = true;
252 Dmsg2(130, "FT_LNK saving: %s -> %s\n", ff_pkt->fname, ff_pkt->link);
255 jcr->num_files_examined--; /* correct file count */
256 return 1; /* not used */
258 Jmsg(jcr, M_INFO, 1, _(" Recursion turned off. Will not descend from %s into %s\n"),
259 ff_pkt->top_fname, ff_pkt->fname);
260 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
263 /* Suppress message for /dev filesystems */
264 if (strncmp(ff_pkt->fname, "/dev/", 5) != 0) {
265 Jmsg(jcr, M_INFO, 1, _(" %s is a different filesystem. Will not descend from %s into %s\n"),
266 ff_pkt->fname, ff_pkt->top_fname, ff_pkt->fname);
268 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
271 Jmsg(jcr, M_INFO, 1, _(" Disallowed filesystem. Will not descend from %s into %s\n"),
272 ff_pkt->top_fname, ff_pkt->fname);
273 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
276 Jmsg(jcr, M_INFO, 1, _(" Disallowed drive type. Will not descend into %s\n"),
280 Dmsg1(130, "FT_DIREND: %s\n", ff_pkt->link);
283 Dmsg1(130, "FT_SPEC saving: %s\n", ff_pkt->fname);
286 Dmsg1(130, "FT_RAW saving: %s\n", ff_pkt->fname);
287 has_file_data = true;
290 Dmsg1(130, "FT_FIFO saving: %s\n", ff_pkt->fname);
294 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not access %s: ERR=%s\n"), ff_pkt->fname,
295 be.bstrerror(ff_pkt->ff_errno));
301 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not follow link %s: ERR=%s\n"),
302 ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
308 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not stat %s: ERR=%s\n"), ff_pkt->fname,
309 be.bstrerror(ff_pkt->ff_errno));
315 Jmsg(jcr, M_SKIPPED, 1, _(" Unchanged file skipped: %s\n"), ff_pkt->fname);
318 Jmsg(jcr, M_NOTSAVED, 0, _(" Archive file not saved: %s\n"), ff_pkt->fname);
322 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not open directory %s: ERR=%s\n"),
323 ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
328 Jmsg(jcr, M_NOTSAVED, 0, _(" Unknown file type %d; not saved: %s\n"),
329 ff_pkt->type, ff_pkt->fname);
334 Dmsg1(130, "bfiled: sending %s to stored\n", ff_pkt->fname);
336 /* Digests and encryption are only useful if there's file data */
339 * Setup for digest handling. If this fails, the digest will be set to NULL
342 if (ff_pkt->flags & FO_MD5) {
343 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_MD5);
344 digest_stream = STREAM_MD5_DIGEST;
346 } else if (ff_pkt->flags & FO_SHA1) {
347 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA1);
348 digest_stream = STREAM_SHA1_DIGEST;
350 } else if (ff_pkt->flags & FO_SHA256) {
351 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA256);
352 digest_stream = STREAM_SHA256_DIGEST;
354 } else if (ff_pkt->flags & FO_SHA512) {
355 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA512);
356 digest_stream = STREAM_SHA512_DIGEST;
359 /* Did digest initialization fail? */
360 if (digest_stream != STREAM_NONE && digest == NULL) {
361 Jmsg(jcr, M_WARNING, 0, _("%s digest initialization failed\n"),
362 stream_to_ascii(digest_stream));
366 * Set up signature digest handling. If this fails, the signature digest will be set to
369 // TODO landonf: We should really only calculate the digest once, for both verification and signing.
371 signing_digest = crypto_digest_new(jcr, signing_algorithm);
373 /* Full-stop if a failure occurred initializing the signature digest */
374 if (signing_digest == NULL) {
375 Jmsg(jcr, M_NOTSAVED, 0, _("%s signature digest initialization failed\n"),
376 stream_to_ascii(signing_algorithm));
382 /* Enable encryption */
383 if (jcr->pki_encrypt) {
384 ff_pkt->flags |= FO_ENCRYPT;
388 /* Initialise the file descriptor we use for data and other streams. */
390 if (ff_pkt->flags & FO_PORTABLE) {
391 set_portable_backup(&ff_pkt->bfd); /* disable Win32 BackupRead() */
393 if (ff_pkt->reader) {
394 if (!set_prog(&ff_pkt->bfd, ff_pkt->reader, jcr)) {
395 Jmsg(jcr, M_FATAL, 0, _("Python reader program \"%s\" not found.\n"),
401 /* Send attributes -- must be done after binit() */
402 if (!encode_and_send_attributes(jcr, ff_pkt, data_stream)) {
406 /* Set up the encryption context and send the session data to the SD */
407 if (has_file_data && jcr->pki_encrypt) {
408 /* Send our header */
409 Dmsg2(100, "Send hdr fi=%ld stream=%d\n", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA);
410 bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA);
412 /* Grow the bsock buffer to fit our message if necessary */
413 if (sizeof_pool_memory(sd->msg) < jcr->pki_session_encoded_size) {
414 sd->msg = realloc_pool_memory(sd->msg, jcr->pki_session_encoded_size);
417 /* Copy our message over and send it */
418 memcpy(sd->msg, jcr->pki_session_encoded, jcr->pki_session_encoded_size);
419 sd->msglen = jcr->pki_session_encoded_size;
420 jcr->JobBytes += sd->msglen;
422 Dmsg1(100, "Send data len=%d\n", sd->msglen);
424 bnet_sig(sd, BNET_EOD);
428 * Open any file with data that we intend to save, then save it.
430 * Note, if is_win32_backup, we must open the Directory so that
431 * the BackupRead will save its permissions and ownership streams.
433 if (ff_pkt->type != FT_LNKSAVED && S_ISREG(ff_pkt->statp.st_mode)) {
435 do_read = !is_portable_backup(&ff_pkt->bfd) || ff_pkt->statp.st_size > 0;
437 do_read = ff_pkt->statp.st_size > 0;
439 } else if (ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO ||
440 (!is_portable_backup(&ff_pkt->bfd) && ff_pkt->type == FT_DIREND)) {
446 if (ff_pkt->type == FT_FIFO) {
447 tid = start_thread_timer(pthread_self(), 60);
451 int noatime = ff_pkt->flags & FO_NOATIME ? O_NOATIME : 0;
452 if (bopen(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY | noatime, 0) < 0) {
453 ff_pkt->ff_errno = errno;
455 Jmsg(jcr, M_NOTSAVED, 0, _(" Cannot open %s: ERR=%s.\n"), ff_pkt->fname,
459 stop_thread_timer(tid);
465 stop_thread_timer(tid);
469 stat = send_data(jcr, data_stream, ff_pkt, digest, signing_digest);
471 if (ff_pkt->flags & FO_CHKCHANGES) {
472 has_file_changed(jcr, ff_pkt);
475 bclose(&ff_pkt->bfd);
482 #ifdef HAVE_DARWIN_OS
483 /* Regular files can have resource forks and Finder Info */
484 if (ff_pkt->type != FT_LNKSAVED && (S_ISREG(ff_pkt->statp.st_mode) &&
485 ff_pkt->flags & FO_HFSPLUS)) {
486 if (ff_pkt->hfsinfo.rsrclength > 0) {
489 if (!bopen_rsrc(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0) < 0) {
490 ff_pkt->ff_errno = errno;
492 Jmsg(jcr, M_NOTSAVED, -1, _(" Cannot open resource fork for %s: ERR=%s.\n"),
493 ff_pkt->fname, be.bstrerror());
495 if (is_bopen(&ff_pkt->bfd)) {
496 bclose(&ff_pkt->bfd);
500 flags = ff_pkt->flags;
501 ff_pkt->flags &= ~(FO_GZIP|FO_SPARSE);
502 if (flags & FO_ENCRYPT) {
503 rsrc_stream = STREAM_ENCRYPTED_MACOS_FORK_DATA;
505 rsrc_stream = STREAM_MACOS_FORK_DATA;
507 stat = send_data(jcr, rsrc_stream, ff_pkt, digest, signing_digest);
508 ff_pkt->flags = flags;
509 bclose(&ff_pkt->bfd);
515 Dmsg1(300, "Saving Finder Info for \"%s\"\n", ff_pkt->fname);
516 bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, STREAM_HFSPLUS_ATTRIBUTES);
517 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
518 memcpy(sd->msg, ff_pkt->hfsinfo.fndrinfo, 32);
521 crypto_digest_update(digest, (uint8_t *)sd->msg, sd->msglen);
523 if (signing_digest) {
524 crypto_digest_update(signing_digest, (uint8_t *)sd->msg, sd->msglen);
527 bnet_sig(sd, BNET_EOD);
531 if (ff_pkt->flags & FO_ACL) {
532 /* Read access ACLs for files, dirs and links */
533 if (!read_and_send_acl(jcr, BACL_TYPE_ACCESS, STREAM_UNIX_ATTRIBUTES_ACCESS_ACL)) {
536 /* Directories can have default ACLs too */
537 if (ff_pkt->type == FT_DIREND && (BACL_CAP & BACL_CAP_DEFAULTS_DIR)) {
538 if (!read_and_send_acl(jcr, BACL_TYPE_DEFAULT, STREAM_UNIX_ATTRIBUTES_DEFAULT_ACL)) {
544 /* Terminate the signing digest and send it to the Storage daemon */
545 if (signing_digest) {
548 if ((sig = crypto_sign_new(jcr)) == NULL) {
549 Jmsg(jcr, M_FATAL, 0, _("Failed to allocate memory for crypto signature.\n"));
553 if (crypto_sign_add_signer(sig, signing_digest, jcr->pki_keypair) == false) {
554 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
558 /* Get signature size */
559 if (crypto_sign_encode(sig, NULL, &size) == false) {
560 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
564 /* Allocate signature data buffer */
565 buf = (uint8_t *)malloc(size);
570 /* Encode signature data */
571 if (crypto_sign_encode(sig, buf, &size) == false) {
572 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
576 /* Send our header */
577 bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, STREAM_SIGNED_DIGEST);
578 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
580 /* Grow the bsock buffer to fit our message if necessary */
581 if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
582 sd->msg = realloc_pool_memory(sd->msg, size);
585 /* Copy our message over and send it */
586 memcpy(sd->msg, buf, size);
589 bnet_sig(sd, BNET_EOD); /* end of checksum */
592 /* Terminate any digest and send it to Storage daemon and the Director */
594 uint8_t md[CRYPTO_DIGEST_MAX_SIZE];
599 if (crypto_digest_finalize(digest, md, &size)) {
600 bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, digest_stream);
601 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
602 memcpy(sd->msg, md, size);
605 bnet_sig(sd, BNET_EOD); /* end of checksum */
610 rtnstat = 1; /* good return */
614 crypto_digest_free(digest);
616 if (signing_digest) {
617 crypto_digest_free(signing_digest);
620 crypto_sign_free(sig);
629 * Send data read from an already open file descriptor.
631 * We return 1 on sucess and 0 on errors.
634 * We use ff_pkt->statp.st_size when FO_SPARSE to know when to stop
636 * Currently this is not a problem as the only other stream, resource forks,
637 * are not handled as sparse files.
639 int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest,
640 DIGEST *signing_digest)
642 BSOCK *sd = jcr->store_bsock;
643 uint64_t fileAddr = 0; /* file address */
645 int32_t rsize = jcr->buf_size; /* read buffer size */
647 CIPHER_CONTEXT *cipher_ctx = NULL; /* Quell bogus uninitialized warnings */
648 const uint8_t *cipher_input;
649 uint32_t cipher_input_len;
650 uint32_t cipher_block_size;
651 uint32_t encrypted_len;
652 #ifdef FD_NO_SEND_TEST
657 rbuf = sd->msg; /* read buffer */
658 wbuf = sd->msg; /* write buffer */
659 cipher_input = (uint8_t *)rbuf; /* encrypt uncompressed data */
661 Dmsg1(300, "Saving data, type=%d\n", ff_pkt->type);
664 uLong compress_len = 0;
665 uLong max_compress_len = 0;
666 const Bytef *cbuf = NULL;
669 if (ff_pkt->flags & FO_GZIP) {
670 if (ff_pkt->flags & FO_SPARSE) {
671 cbuf = (Bytef *)jcr->compress_buf + SPARSE_FADDR_SIZE;
672 max_compress_len = jcr->compress_buf_size - SPARSE_FADDR_SIZE;
674 cbuf = (Bytef *)jcr->compress_buf;
675 max_compress_len = jcr->compress_buf_size; /* set max length */
677 wbuf = jcr->compress_buf; /* compressed output here */
678 cipher_input = (uint8_t *)jcr->compress_buf; /* encrypt compressed data */
681 * Only change zlib parameters if there is no pending operation.
682 * This should never happen as deflatereset is called after each
686 if (((z_stream*)jcr->pZLIB_compress_workset)->total_in == 0) {
687 /* set gzip compression level - must be done per file */
688 if ((zstat=deflateParams((z_stream*)jcr->pZLIB_compress_workset,
689 ff_pkt->GZIP_level, Z_DEFAULT_STRATEGY)) != Z_OK) {
690 Jmsg(jcr, M_FATAL, 0, _("Compression deflateParams error: %d\n"), zstat);
691 set_jcr_job_status(jcr, JS_ErrorTerminated);
697 const uint32_t max_compress_len = 0;
700 if (ff_pkt->flags & FO_ENCRYPT) {
701 if (ff_pkt->flags & FO_SPARSE) {
702 Jmsg0(jcr, M_FATAL, 0, _("Encrypting sparse data not supported.\n"));
705 /* Allocate the cipher context */
706 if ((cipher_ctx = crypto_cipher_new(jcr->pki_session, true,
707 &cipher_block_size)) == NULL) {
708 /* Shouldn't happen! */
709 Jmsg0(jcr, M_FATAL, 0, _("Failed to initialize encryption context.\n"));
714 * Grow the crypto buffer, if necessary.
715 * crypto_cipher_update() will buffer up to (cipher_block_size - 1).
716 * We grow crypto_buf to the maximum number of blocks that
717 * could be returned for the given read buffer size.
718 * (Using the larger of either rsize or max_compress_len)
720 jcr->crypto_buf = check_pool_memory_size(jcr->crypto_buf,
721 (MAX(rsize + (int)sizeof(uint32_t), (int32_t)max_compress_len) +
722 cipher_block_size - 1) / cipher_block_size * cipher_block_size);
724 wbuf = jcr->crypto_buf; /* Encrypted, possibly compressed output here. */
728 * Send Data header to Storage daemon
729 * <file-index> <stream> <info>
731 if (!bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, stream)) {
732 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
736 Dmsg1(300, ">stored: datahdr %s\n", sd->msg);
739 * Make space at beginning of buffer for fileAddr because this
740 * same buffer will be used for writing if compression is off.
742 if (ff_pkt->flags & FO_SPARSE) {
743 rbuf += SPARSE_FADDR_SIZE;
744 rsize -= SPARSE_FADDR_SIZE;
745 #ifdef HAVE_FREEBSD_OS
747 * To read FreeBSD partitions, the read size must be
750 rsize = (rsize/512) * 512;
754 /* a RAW device read on win32 only works if the buffer is a multiple of 512 */
756 if (S_ISBLK(ff_pkt->statp.st_mode))
757 rsize = (rsize/512) * 512;
763 while ((sd->msglen=(uint32_t)bread(&ff_pkt->bfd, rbuf, rsize)) > 0) {
765 /* Check for sparse blocks */
766 if (ff_pkt->flags & FO_SPARSE) {
768 bool haveBlock = true;
769 if (sd->msglen == rsize &&
770 fileAddr+sd->msglen < (uint64_t)ff_pkt->statp.st_size ||
771 ((ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO) &&
772 (uint64_t)ff_pkt->statp.st_size == 0)) {
773 haveBlock = !is_buf_zero(rbuf, rsize);
776 ser_begin(wbuf, SPARSE_FADDR_SIZE);
777 ser_uint64(fileAddr); /* store fileAddr in begin of buffer */
779 fileAddr += sd->msglen; /* update file address */
781 continue; /* skip block of zeros */
785 jcr->ReadBytes += sd->msglen; /* count bytes read */
787 /* Uncompressed cipher input length */
788 cipher_input_len = sd->msglen;
790 /* Update checksum if requested */
792 crypto_digest_update(digest, (uint8_t *)rbuf, sd->msglen);
795 /* Update signing digest if requested */
796 if (signing_digest) {
797 crypto_digest_update(signing_digest, (uint8_t *)rbuf, sd->msglen);
801 /* Do compression if turned on */
802 if (ff_pkt->flags & FO_GZIP && jcr->pZLIB_compress_workset) {
803 Dmsg3(400, "cbuf=0x%x rbuf=0x%x len=%u\n", cbuf, rbuf, sd->msglen);
805 ((z_stream*)jcr->pZLIB_compress_workset)->next_in = (Bytef *)rbuf;
806 ((z_stream*)jcr->pZLIB_compress_workset)->avail_in = sd->msglen;
807 ((z_stream*)jcr->pZLIB_compress_workset)->next_out = (Bytef *)cbuf;
808 ((z_stream*)jcr->pZLIB_compress_workset)->avail_out = max_compress_len;
810 if ((zstat=deflate((z_stream*)jcr->pZLIB_compress_workset, Z_FINISH)) != Z_STREAM_END) {
811 Jmsg(jcr, M_FATAL, 0, _("Compression deflate error: %d\n"), zstat);
812 set_jcr_job_status(jcr, JS_ErrorTerminated);
815 compress_len = ((z_stream*)jcr->pZLIB_compress_workset)->total_out;
816 /* reset zlib stream to be able to begin from scratch again */
817 if ((zstat=deflateReset((z_stream*)jcr->pZLIB_compress_workset)) != Z_OK) {
818 Jmsg(jcr, M_FATAL, 0, _("Compression deflateReset error: %d\n"), zstat);
819 set_jcr_job_status(jcr, JS_ErrorTerminated);
823 Dmsg2(400, "compressed len=%d uncompressed len=%d\n", compress_len,
826 sd->msglen = compress_len; /* set compressed length */
827 cipher_input_len = compress_len;
831 * Note, here we prepend the current record length to the beginning
832 * of the encrypted data. This is because both sparse and compression
833 * restore handling want records returned to them with exactly the
834 * same number of bytes that were processed in the backup handling.
835 * That is, both are block filters rather than a stream. When doing
836 * compression, the compression routines may buffer data, so that for
837 * any one record compressed, when it is decompressed the same size
838 * will not be obtained. Of course, the buffered data eventually comes
839 * out in subsequent crypto_cipher_update() calls or at least
840 * when crypto_cipher_finalize() is called. Unfortunately, this
841 * "feature" of encryption enormously complicates the restore code.
843 if (ff_pkt->flags & FO_ENCRYPT) {
844 uint32_t initial_len = 0;
847 if (ff_pkt->flags & FO_SPARSE) {
848 cipher_input_len += SPARSE_FADDR_SIZE;
851 /* Encrypt the length of the input block */
852 uint8_t packet_len[sizeof(uint32_t)];
854 ser_begin(packet_len, sizeof(uint32_t));
855 ser_uint32(cipher_input_len); /* store data len in begin of buffer */
856 Dmsg1(20, "Encrypt len=%d\n", cipher_input_len);
858 if (!crypto_cipher_update(cipher_ctx, packet_len, sizeof(packet_len),
859 (u_int8_t *)jcr->crypto_buf, &initial_len)) {
860 /* Encryption failed. Shouldn't happen. */
861 Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
865 /* Encrypt the input block */
866 if (crypto_cipher_update(cipher_ctx, cipher_input, cipher_input_len,
867 (u_int8_t *)&jcr->crypto_buf[initial_len], &encrypted_len)) {
868 if ((initial_len + encrypted_len) == 0) {
869 /* No full block of data available, read more data */
872 Dmsg2(400, "encrypted len=%d unencrypted len=%d\n", encrypted_len,
874 sd->msglen = initial_len + encrypted_len; /* set encrypted length */
876 /* Encryption failed. Shouldn't happen. */
877 Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
882 /* Send the buffer to the Storage daemon */
883 if (ff_pkt->flags & FO_SPARSE) {
884 sd->msglen += SPARSE_FADDR_SIZE; /* include fileAddr in size */
886 sd->msg = wbuf; /* set correct write buffer */
887 if (!bnet_send(sd)) {
888 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
892 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
894 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
895 sd->msg = msgsave; /* restore read buffer */
897 } /* end while read file data */
899 if (sd->msglen < 0) { /* error */
901 Jmsg(jcr, M_ERROR, 0, _("Read error on file %s. ERR=%s\n"),
902 ff_pkt->fname, be.bstrerror(ff_pkt->bfd.berrno));
903 if (jcr->Errors++ > 1000) { /* insanity check */
904 Jmsg(jcr, M_FATAL, 0, _("Too many errors.\n"));
906 } else if (ff_pkt->flags & FO_ENCRYPT) {
908 * For encryption, we must call finalize to push out any
911 if (!crypto_cipher_finalize(cipher_ctx, (uint8_t *)jcr->crypto_buf,
913 /* Padding failed. Shouldn't happen. */
914 Jmsg(jcr, M_FATAL, 0, _("Encryption padding error\n"));
918 /* Note, on SSL pre-0.9.7, there is always some output */
919 if (encrypted_len > 0) {
920 sd->msglen = encrypted_len; /* set encrypted length */
921 sd->msg = jcr->crypto_buf; /* set correct write buffer */
922 if (!bnet_send(sd)) {
923 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
927 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
928 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
929 sd->msg = msgsave; /* restore bnet buffer */
933 if (!bnet_sig(sd, BNET_EOD)) { /* indicate end of file data */
934 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
939 /* Free the cipher context */
941 crypto_cipher_free(cipher_ctx);
946 /* Free the cipher context */
948 crypto_cipher_free(cipher_ctx);
951 sd->msg = msgsave; /* restore bnet buffer */
957 * Read and send an ACL for the last encountered file.
959 static bool read_and_send_acl(JCR *jcr, int acltype, int stream)
962 BSOCK *sd = jcr->store_bsock;
965 #ifdef FD_NO_SEND_TEST
969 len = bacl_get(jcr, acltype);
971 Jmsg1(jcr, M_WARNING, 0, _("Error reading ACL of %s\n"), jcr->last_fname);
975 return true; /* no ACL */
979 if (!bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, stream)) {
980 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
985 /* Send the buffer to the storage deamon */
986 Dmsg2(400, "Backing up ACL type 0x%2x <%s>\n", acltype, jcr->acl_text);
988 sd->msg = jcr->acl_text;
989 sd->msglen = len + 1;
990 if (!bnet_send(sd)) {
993 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
998 jcr->JobBytes += sd->msglen;
1000 if (!bnet_sig(sd, BNET_EOD)) {
1001 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1006 Dmsg1(200, "ACL of file: %s successfully backed up!\n", jcr->last_fname);
1011 static bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream)
1013 BSOCK *sd = jcr->store_bsock;
1014 char attribs[MAXSTRING];
1015 char attribsEx[MAXSTRING];
1018 #ifdef FD_NO_SEND_TEST
1022 Dmsg1(300, "encode_and_send_attrs fname=%s\n", ff_pkt->fname);
1023 /* Find what data stream we will use, then encode the attributes */
1024 if ((data_stream = select_data_stream(ff_pkt)) == STREAM_NONE) {
1025 /* This should not happen */
1026 Jmsg0(jcr, M_FATAL, 0, _("Invalid file flags, no supported data stream type.\n"));
1029 encode_stat(attribs, ff_pkt, data_stream);
1031 /* Now possibly extend the attributes */
1032 attr_stream = encode_attribsEx(jcr, attribsEx, ff_pkt);
1034 Dmsg3(300, "File %s\nattribs=%s\nattribsEx=%s\n", ff_pkt->fname, attribs, attribsEx);
1037 jcr->JobFiles++; /* increment number of files sent */
1038 ff_pkt->FileIndex = jcr->JobFiles; /* return FileIndex */
1039 pm_strcpy(jcr->last_fname, ff_pkt->fname);
1043 * Send Attributes header to Storage daemon
1044 * <file-index> <stream> <info>
1046 if (!bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, attr_stream)) {
1047 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1051 Dmsg1(300, ">stored: attrhdr %s\n", sd->msg);
1054 * Send file attributes to Storage daemon
1057 * Filename (full path)
1058 * Encoded attributes
1059 * Link name (if type==FT_LNK or FT_LNKSAVED)
1060 * Encoded extended-attributes (for Win32)
1062 * For a directory, link is the same as fname, but with trailing
1063 * slash. For a linked file, link is the link.
1066 if (ff_pkt->type == FT_LNK || ff_pkt->type == FT_LNKSAVED) {
1067 Dmsg2(300, "Link %s to %s\n", ff_pkt->fname, ff_pkt->link);
1068 stat = bnet_fsend(sd, "%ld %d %s%c%s%c%s%c%s%c", jcr->JobFiles,
1069 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, ff_pkt->link, 0,
1071 } else if (ff_pkt->type == FT_DIREND) {
1072 /* Here link is the canonical filename (i.e. with trailing slash) */
1073 stat = bnet_fsend(sd, "%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
1074 ff_pkt->type, ff_pkt->link, 0, attribs, 0, 0, attribsEx, 0);
1076 stat = bnet_fsend(sd, "%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
1077 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, 0, attribsEx, 0);
1079 unstrip_path(ff_pkt);
1081 Dmsg2(300, ">stored: attr len=%d: %s\n", sd->msglen, sd->msg);
1083 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1087 bnet_sig(sd, BNET_EOD); /* indicate end of attributes data */
1092 * Do in place strip of path
1094 static bool do_strip(int count, char *in)
1100 /* Copy to first path separator -- Win32 might have c: ... */
1101 while (*in && !IsPathSeparator(*in)) {
1105 numsep++; /* one separator seen */
1106 for (stripped=0; stripped<count && *in; stripped++) {
1107 while (*in && !IsPathSeparator(*in)) {
1108 in++; /* skip chars */
1111 numsep++; /* count separators seen */
1112 in++; /* skip separator */
1116 while (*in) { /* copy to end */
1117 if (IsPathSeparator(*in)) {
1123 Dmsg4(500, "stripped=%d count=%d numsep=%d sep>count=%d\n",
1124 stripped, count, numsep, numsep>count);
1125 return stripped==count && numsep>count;
1129 * If requested strip leading components of the path
1131 static void strip_path(FF_PKT *ff_pkt)
1133 if (!(ff_pkt->flags & FO_STRIPPATH) || ff_pkt->strip_path <= 0) {
1134 Dmsg1(200, "No strip for %s\n", ff_pkt->fname);
1137 if (!ff_pkt->fname_save) {
1138 ff_pkt->fname_save = get_pool_memory(PM_FNAME);
1139 ff_pkt->link_save = get_pool_memory(PM_FNAME);
1141 pm_strcpy(ff_pkt->fname_save, ff_pkt->fname);
1144 * Strip path. If it doesn't succeed put it back. If
1145 * it does, and there is a different link string,
1146 * attempt to strip the link. If it fails, but them
1148 * I.e. if either stripping fails don't strip anything.
1150 if (do_strip(ff_pkt->strip_path, ff_pkt->fname)) {
1151 if (ff_pkt->fname != ff_pkt->link) {
1152 pm_strcpy(ff_pkt->link_save, ff_pkt->link);
1153 if (!do_strip(ff_pkt->strip_path, ff_pkt->link)) {
1154 strcpy(ff_pkt->link, ff_pkt->link_save);
1155 strcpy(ff_pkt->fname, ff_pkt->fname_save);
1159 strcpy(ff_pkt->fname, ff_pkt->fname_save);
1161 Dmsg2(200, "fname=%s stripped=%s\n", ff_pkt->fname_save, ff_pkt->fname);
1164 static void unstrip_path(FF_PKT *ff_pkt)
1166 if (!(ff_pkt->flags & FO_STRIPPATH) || ff_pkt->strip_path <= 0) {
1169 strcpy(ff_pkt->fname, ff_pkt->fname_save);
1170 if (ff_pkt->fname != ff_pkt->link) {
1171 strcpy(ff_pkt->link, ff_pkt->link_save);