2 * Bacula File Daemon backup.c send file attributes and data
3 * to the Storage daemon.
5 * Kern Sibbald, March MM
11 Bacula® - The Network Backup Solution
13 Copyright (C) 2000-2007 Free Software Foundation Europe e.V.
15 The main author of Bacula is Kern Sibbald, with contributions from
16 many others, a complete list can be found in the file AUTHORS.
17 This program is Free Software; you can redistribute it and/or
18 modify it under the terms of version two of the GNU General Public
19 License as published by the Free Software Foundation plus additions
20 that are listed in the file LICENSE.
22 This program is distributed in the hope that it will be useful, but
23 WITHOUT ANY WARRANTY; without even the implied warranty of
24 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
25 General Public License for more details.
27 You should have received a copy of the GNU General Public License
28 along with this program; if not, write to the Free Software
29 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
32 Bacula® is a registered trademark of John Walker.
33 The licensor of Bacula is the Free Software Foundation Europe
34 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
35 Switzerland, email:ftf@fsfeurope.org.
41 /* Forward referenced functions */
42 static int save_file(FF_PKT *ff_pkt, void *pkt, bool top_level);
43 static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, DIGEST *signature_digest);
44 static bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream);
45 static bool read_and_send_acl(JCR *jcr, int acltype, int stream);
48 * Find all the requested files and send them
49 * to the Storage daemon.
51 * Note, we normally carry on a one-way
52 * conversation from this point on with the SD, simply blasting
53 * data to him. To properly know what is going on, we
54 * also run a "heartbeat" monitor which reads the socket and
55 * reacts accordingly (at the moment it has nothing to do
56 * except echo the heartbeat to the Director).
59 bool blast_data_to_storage_daemon(JCR *jcr, char *addr)
63 // TODO landonf: Allow user to specify encryption algorithm
64 crypto_cipher_t cipher = CRYPTO_CIPHER_AES_128_CBC;
67 sd = jcr->store_bsock;
69 set_jcr_job_status(jcr, JS_Running);
71 Dmsg1(300, "bfiled: opened data connection %d to stored\n", sd->fd);
74 CLIENT *client = (CLIENT *)GetNextRes(R_CLIENT, NULL);
78 buf_size = client->max_network_buffer_size;
80 buf_size = 0; /* use default */
82 if (!bnet_set_buffer_size(sd, buf_size, BNET_SETBUF_WRITE)) {
83 set_jcr_job_status(jcr, JS_ErrorTerminated);
84 Jmsg(jcr, M_FATAL, 0, _("Cannot set buffer size FD->SD.\n"));
88 jcr->buf_size = sd->msglen;
89 /* Adjust for compression so that output buffer is
90 * 12 bytes + 0.1% larger than input buffer plus 18 bytes.
91 * This gives a bit extra plus room for the sparse addr if any.
92 * Note, we adjust the read size to be smaller so that the
93 * same output buffer can be used without growing it.
95 * The zlib compression workset is initialized here to minimise
96 * the "per file" load. The jcr member is only set, if the init was successful.
98 jcr->compress_buf_size = jcr->buf_size + ((jcr->buf_size+999) / 1000) + 30;
99 jcr->compress_buf = get_memory(jcr->compress_buf_size);
102 z_stream *pZlibStream = (z_stream*)malloc(sizeof(z_stream));
104 pZlibStream->zalloc = Z_NULL;
105 pZlibStream->zfree = Z_NULL;
106 pZlibStream->opaque = Z_NULL;
107 pZlibStream->state = Z_NULL;
109 if (deflateInit(pZlibStream, Z_DEFAULT_COMPRESSION) == Z_OK)
110 jcr->pZLIB_compress_workset = pZlibStream;
116 /* Create encryption session data and a cached, DER-encoded session data
117 * structure. We use a single session key for each backup, so we'll encode
118 * the session data only once. */
119 if (jcr->pki_encrypt) {
122 /* Create per-job session encryption context */
123 jcr->pki_session = crypto_session_new(cipher, jcr->pki_recipients);
125 /* Get the session data size */
126 if (crypto_session_encode(jcr->pki_session, (uint8_t *)0, &size) == false) {
127 Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n"));
131 /* Allocate buffer */
132 jcr->pki_session_encoded = (uint8_t *)malloc(size);
133 if (!jcr->pki_session_encoded) {
137 /* Encode session data */
138 if (crypto_session_encode(jcr->pki_session, jcr->pki_session_encoded, &size) == false) {
139 Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n"));
143 /* ... and store the encoded size */
144 jcr->pki_session_encoded_size = size;
146 /* Allocate the encryption/decryption buffer */
147 jcr->crypto_buf = get_memory(CRYPTO_CIPHER_MAX_BLOCK_SIZE);
150 Dmsg1(300, "set_find_options ff=%p\n", jcr->ff);
151 set_find_options((FF_PKT *)jcr->ff, jcr->incremental, jcr->mtime);
152 Dmsg0(300, "start find files\n");
154 start_heartbeat_monitor(jcr);
156 jcr->acl_text = get_pool_memory(PM_MESSAGE);
158 /* Subroutine save_file() is called for each file */
159 if (!find_files(jcr, (FF_PKT *)jcr->ff, save_file, (void *)jcr)) {
160 ok = false; /* error */
161 set_jcr_job_status(jcr, JS_ErrorTerminated);
162 // Jmsg(jcr, M_FATAL, 0, _("Find files error.\n"));
165 free_pool_memory(jcr->acl_text);
167 stop_heartbeat_monitor(jcr);
169 bnet_sig(sd, BNET_EOD); /* end of sending data */
175 if (jcr->compress_buf) {
176 free_pool_memory(jcr->compress_buf);
177 jcr->compress_buf = NULL;
179 if (jcr->pZLIB_compress_workset) {
180 /* Free the zlib stream */
182 deflateEnd((z_stream *)jcr->pZLIB_compress_workset);
184 free (jcr->pZLIB_compress_workset);
185 jcr->pZLIB_compress_workset = NULL;
187 if (jcr->crypto_buf) {
188 free_pool_memory(jcr->crypto_buf);
189 jcr->crypto_buf = NULL;
191 if (jcr->pki_session) {
192 crypto_session_free(jcr->pki_session);
194 if (jcr->pki_session_encoded) {
195 free(jcr->pki_session_encoded);
198 Dmsg1(100, "end blast_data ok=%d\n", ok);
203 * Called here by find() for each file included.
204 * This is a callback. The original is find_files() above.
206 * Send the file and its data to the Storage daemon.
210 * -1 to ignore file/directory (not used here)
212 static int save_file(FF_PKT *ff_pkt, void *vjcr, bool top_level)
214 bool do_read = false;
215 int stat, data_stream;
217 DIGEST *digest = NULL;
218 DIGEST *signing_digest = NULL;
219 int digest_stream = STREAM_NONE;
220 SIGNATURE *sig = NULL;
222 bool has_file_data = false;
223 // TODO landonf: Allow the user to specify the digest algorithm
225 crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA256;
227 crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA1;
229 JCR *jcr = (JCR *)vjcr;
230 BSOCK *sd = jcr->store_bsock;
232 if (job_canceled(jcr)) {
236 jcr->num_files_examined++; /* bump total file count */
238 switch (ff_pkt->type) {
239 case FT_LNKSAVED: /* Hard linked, file already saved */
240 Dmsg2(130, "FT_LNKSAVED hard link: %s => %s\n", ff_pkt->fname, ff_pkt->link);
243 Dmsg1(130, "FT_REGE saving: %s\n", ff_pkt->fname);
244 has_file_data = true;
247 Dmsg1(130, "FT_REG saving: %s\n", ff_pkt->fname);
248 has_file_data = true;
251 Dmsg2(130, "FT_LNK saving: %s -> %s\n", ff_pkt->fname, ff_pkt->link);
254 jcr->num_files_examined--; /* correct file count */
255 return 1; /* not used */
257 Jmsg(jcr, M_INFO, 1, _(" Recursion turned off. Will not descend from %s into %s\n"),
258 ff_pkt->top_fname, ff_pkt->fname);
259 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
262 /* Suppress message for /dev filesystems */
263 if (strncmp(ff_pkt->fname, "/dev/", 5) != 0) {
264 Jmsg(jcr, M_INFO, 1, _(" %s is a different filesystem. Will not descend from %s into %s\n"),
265 ff_pkt->fname, ff_pkt->top_fname, ff_pkt->fname);
267 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
270 Jmsg(jcr, M_INFO, 1, _(" Disallowed filesystem. Will not descend from %s into %s\n"),
271 ff_pkt->top_fname, ff_pkt->fname);
272 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
275 Jmsg(jcr, M_INFO, 1, _(" Disallowed drive type. Will not descend into %s\n"),
279 Dmsg1(130, "FT_DIREND: %s\n", ff_pkt->link);
282 Dmsg1(130, "FT_SPEC saving: %s\n", ff_pkt->fname);
285 Dmsg1(130, "FT_RAW saving: %s\n", ff_pkt->fname);
286 has_file_data = true;
289 Dmsg1(130, "FT_FIFO saving: %s\n", ff_pkt->fname);
293 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not access %s: ERR=%s\n"), ff_pkt->fname,
294 be.strerror(ff_pkt->ff_errno));
300 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not follow link %s: ERR=%s\n"), ff_pkt->fname,
301 be.strerror(ff_pkt->ff_errno));
307 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not stat %s: ERR=%s\n"), ff_pkt->fname,
308 be.strerror(ff_pkt->ff_errno));
314 Jmsg(jcr, M_SKIPPED, 1, _(" Unchanged file skipped: %s\n"), ff_pkt->fname);
317 Jmsg(jcr, M_NOTSAVED, 0, _(" Archive file not saved: %s\n"), ff_pkt->fname);
321 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not open directory %s: ERR=%s\n"), ff_pkt->fname,
322 be.strerror(ff_pkt->ff_errno));
327 Jmsg(jcr, M_NOTSAVED, 0, _(" Unknown file type %d; not saved: %s\n"), ff_pkt->type, ff_pkt->fname);
332 Dmsg1(130, "bfiled: sending %s to stored\n", ff_pkt->fname);
334 /* Digests and encryption are only useful if there's file data */
337 * Setup for digest handling. If this fails, the digest will be set to NULL
340 if (ff_pkt->flags & FO_MD5) {
341 digest = crypto_digest_new(CRYPTO_DIGEST_MD5);
342 digest_stream = STREAM_MD5_DIGEST;
344 } else if (ff_pkt->flags & FO_SHA1) {
345 digest = crypto_digest_new(CRYPTO_DIGEST_SHA1);
346 digest_stream = STREAM_SHA1_DIGEST;
348 } else if (ff_pkt->flags & FO_SHA256) {
349 digest = crypto_digest_new(CRYPTO_DIGEST_SHA256);
350 digest_stream = STREAM_SHA256_DIGEST;
352 } else if (ff_pkt->flags & FO_SHA512) {
353 digest = crypto_digest_new(CRYPTO_DIGEST_SHA512);
354 digest_stream = STREAM_SHA512_DIGEST;
357 /* Did digest initialization fail? */
358 if (digest_stream != STREAM_NONE && digest == NULL) {
359 Jmsg(jcr, M_WARNING, 0, _("%s digest initialization failed\n"),
360 stream_to_ascii(digest_stream));
364 * Set up signature digest handling. If this fails, the signature digest will be set to
367 // TODO landonf: We should really only calculate the digest once, for both verification and signing.
369 signing_digest = crypto_digest_new(signing_algorithm);
371 /* Full-stop if a failure occurred initializing the signature digest */
372 if (signing_digest == NULL) {
373 Jmsg(jcr, M_NOTSAVED, 0, _("%s signature digest initialization failed\n"),
374 stream_to_ascii(signing_algorithm));
380 /* Enable encryption */
381 if (jcr->pki_encrypt) {
382 ff_pkt->flags |= FO_ENCRYPT;
386 /* Initialise the file descriptor we use for data and other streams. */
388 if (ff_pkt->flags & FO_PORTABLE) {
389 set_portable_backup(&ff_pkt->bfd); /* disable Win32 BackupRead() */
391 if (ff_pkt->reader) {
392 if (!set_prog(&ff_pkt->bfd, ff_pkt->reader, jcr)) {
393 Jmsg(jcr, M_FATAL, 0, _("Python reader program \"%s\" not found.\n"),
399 /* Send attributes -- must be done after binit() */
400 if (!encode_and_send_attributes(jcr, ff_pkt, data_stream)) {
405 * Open any file with data that we intend to save, then save it.
407 * Note, if is_win32_backup, we must open the Directory so that
408 * the BackupRead will save its permissions and ownership streams.
411 if (ff_pkt->type != FT_LNKSAVED && S_ISREG(ff_pkt->statp.st_mode)) {
413 do_read = !is_portable_backup(&ff_pkt->bfd) || ff_pkt->statp.st_size > 0;
415 do_read = ff_pkt->statp.st_size > 0;
417 } else if (ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO ||
418 (!is_portable_backup(&ff_pkt->bfd) && ff_pkt->type == FT_DIREND)) {
424 if (ff_pkt->type == FT_FIFO) {
425 tid = start_thread_timer(pthread_self(), 60);
429 int noatime = ff_pkt->flags & FO_NOATIME ? O_NOATIME : 0;
430 if (bopen(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY | noatime, 0) < 0) {
431 ff_pkt->ff_errno = errno;
433 Jmsg(jcr, M_NOTSAVED, 0, _(" Cannot open %s: ERR=%s.\n"), ff_pkt->fname,
437 stop_thread_timer(tid);
443 stop_thread_timer(tid);
447 /* Set up the encryption context, send the session data to the SD */
448 if (jcr->pki_encrypt) {
449 /* Send our header */
450 Dmsg2(100, "Send hdr fi=%ld stream=%d\n", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA);
451 bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA);
453 /* Grow the bsock buffer to fit our message if necessary */
454 if (sizeof_pool_memory(sd->msg) < jcr->pki_session_encoded_size) {
455 sd->msg = realloc_pool_memory(sd->msg, jcr->pki_session_encoded_size);
458 /* Copy our message over and send it */
459 memcpy(sd->msg, jcr->pki_session_encoded, jcr->pki_session_encoded_size);
460 sd->msglen = jcr->pki_session_encoded_size;
461 jcr->JobBytes += sd->msglen;
463 Dmsg1(100, "Send data len=%d\n", sd->msglen);
465 bnet_sig(sd, BNET_EOD);
468 stat = send_data(jcr, data_stream, ff_pkt, digest, signing_digest);
469 bclose(&ff_pkt->bfd);
475 #ifdef HAVE_DARWIN_OS
476 /* Regular files can have resource forks and Finder Info */
477 if (ff_pkt->type != FT_LNKSAVED && (S_ISREG(ff_pkt->statp.st_mode) &&
478 ff_pkt->flags & FO_HFSPLUS)) {
479 if (ff_pkt->hfsinfo.rsrclength > 0) {
482 if (!bopen_rsrc(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0) < 0) {
483 ff_pkt->ff_errno = errno;
485 Jmsg(jcr, M_NOTSAVED, -1, _(" Cannot open resource fork for %s: ERR=%s.\n"), ff_pkt->fname,
488 if (is_bopen(&ff_pkt->bfd)) {
489 bclose(&ff_pkt->bfd);
493 flags = ff_pkt->flags;
494 ff_pkt->flags &= ~(FO_GZIP|FO_SPARSE);
495 if (flags & FO_ENCRYPT) {
496 rsrc_stream = STREAM_ENCRYPTED_MACOS_FORK_DATA;
498 rsrc_stream = STREAM_MACOS_FORK_DATA;
500 stat = send_data(jcr, rsrc_stream, ff_pkt, digest, signing_digest);
501 ff_pkt->flags = flags;
502 bclose(&ff_pkt->bfd);
508 Dmsg1(300, "Saving Finder Info for \"%s\"\n", ff_pkt->fname);
509 bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, STREAM_HFSPLUS_ATTRIBUTES);
510 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
511 memcpy(sd->msg, ff_pkt->hfsinfo.fndrinfo, 32);
514 crypto_digest_update(digest, (uint8_t *)sd->msg, sd->msglen);
516 if (signing_digest) {
517 crypto_digest_update(signing_digest, (uint8_t *)sd->msg, sd->msglen);
520 bnet_sig(sd, BNET_EOD);
524 if (ff_pkt->flags & FO_ACL) {
525 /* Read access ACLs for files, dirs and links */
526 if (!read_and_send_acl(jcr, BACL_TYPE_ACCESS, STREAM_UNIX_ATTRIBUTES_ACCESS_ACL)) {
529 /* Directories can have default ACLs too */
530 if (ff_pkt->type == FT_DIREND && (BACL_CAP & BACL_CAP_DEFAULTS_DIR)) {
531 if (!read_and_send_acl(jcr, BACL_TYPE_DEFAULT, STREAM_UNIX_ATTRIBUTES_DEFAULT_ACL)) {
537 /* Terminate the signing digest and send it to the Storage daemon */
538 if (signing_digest) {
541 if ((sig = crypto_sign_new()) == NULL) {
542 Jmsg(jcr, M_FATAL, 0, _("Failed to allocate memory for stream signature.\n"));
546 if (crypto_sign_add_signer(sig, signing_digest, jcr->pki_keypair) == false) {
547 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
551 /* Get signature size */
552 if (crypto_sign_encode(sig, NULL, &size) == false) {
553 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
557 /* Allocate signature data buffer */
558 buf = (uint8_t *)malloc(size);
563 /* Encode signature data */
564 if (crypto_sign_encode(sig, buf, &size) == false) {
565 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
569 /* Send our header */
570 bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, STREAM_SIGNED_DIGEST);
571 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
573 /* Grow the bsock buffer to fit our message if necessary */
574 if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
575 sd->msg = realloc_pool_memory(sd->msg, size);
578 /* Copy our message over and send it */
579 memcpy(sd->msg, buf, size);
582 bnet_sig(sd, BNET_EOD); /* end of checksum */
586 /* Terminate any digest and send it to Storage daemon and the Director */
588 uint8_t md[CRYPTO_DIGEST_MAX_SIZE];
593 if (crypto_digest_finalize(digest, md, &size)) {
594 bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, digest_stream);
595 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
596 memcpy(sd->msg, md, size);
599 bnet_sig(sd, BNET_EOD); /* end of checksum */
604 rtnstat = 1; /* good return */
608 crypto_digest_free(digest);
610 if (signing_digest) {
611 crypto_digest_free(signing_digest);
614 crypto_sign_free(sig);
623 * Send data read from an already open file descriptor.
625 * We return 1 on sucess and 0 on errors.
628 * We use ff_pkt->statp.st_size when FO_SPARSE to know when to stop
630 * Currently this is not a problem as the only other stream, resource forks,
631 * are not handled as sparse files.
633 int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest,
634 DIGEST *signing_digest)
636 BSOCK *sd = jcr->store_bsock;
637 uint64_t fileAddr = 0; /* file address */
639 int32_t rsize = jcr->buf_size; /* read buffer size */
641 CIPHER_CONTEXT *cipher_ctx = NULL; /* Quell bogus uninitialized warnings */
642 const uint8_t *cipher_input;
643 uint32_t cipher_input_len;
644 uint32_t cipher_block_size;
645 uint32_t encrypted_len;
646 #ifdef FD_NO_SEND_TEST
651 rbuf = sd->msg; /* read buffer */
652 wbuf = sd->msg; /* write buffer */
653 cipher_input = (uint8_t *)rbuf; /* encrypt uncompressed data */
655 Dmsg1(300, "Saving data, type=%d\n", ff_pkt->type);
658 uLong compress_len = 0;
659 uLong max_compress_len = 0;
660 const Bytef *cbuf = NULL;
663 if (ff_pkt->flags & FO_GZIP) {
664 if (ff_pkt->flags & FO_SPARSE) {
665 cbuf = (Bytef *)jcr->compress_buf + SPARSE_FADDR_SIZE;
666 max_compress_len = jcr->compress_buf_size - SPARSE_FADDR_SIZE;
668 cbuf = (Bytef *)jcr->compress_buf;
669 max_compress_len = jcr->compress_buf_size; /* set max length */
671 wbuf = jcr->compress_buf; /* compressed output here */
672 cipher_input = (uint8_t *)jcr->compress_buf; /* encrypt compressed data */
675 * Only change zlib parameters if there is no pending operation.
676 * This should never happen as deflatereset is called after each
680 if (((z_stream*)jcr->pZLIB_compress_workset)->total_in == 0) {
681 /* set gzip compression level - must be done per file */
682 if ((zstat=deflateParams((z_stream*)jcr->pZLIB_compress_workset,
683 ff_pkt->GZIP_level, Z_DEFAULT_STRATEGY)) != Z_OK) {
684 Jmsg(jcr, M_FATAL, 0, _("Compression deflateParams error: %d\n"), zstat);
685 set_jcr_job_status(jcr, JS_ErrorTerminated);
691 const uint32_t max_compress_len = 0;
694 if (ff_pkt->flags & FO_ENCRYPT) {
695 if (ff_pkt->flags & FO_SPARSE) {
696 Jmsg0(jcr, M_FATAL, 0, _("Encrypting sparse data not supported.\n"));
699 /* Allocate the cipher context */
700 if ((cipher_ctx = crypto_cipher_new(jcr->pki_session, true,
701 &cipher_block_size)) == NULL) {
702 /* Shouldn't happen! */
703 Jmsg0(jcr, M_FATAL, 0, _("Failed to initialize encryption context.\n"));
708 * Grow the crypto buffer, if necessary.
709 * crypto_cipher_update() will buffer up to (cipher_block_size - 1).
710 * We grow crypto_buf to the maximum number of blocks that
711 * could be returned for the given read buffer size.
712 * (Using the larger of either rsize or max_compress_len)
714 jcr->crypto_buf = check_pool_memory_size(jcr->crypto_buf,
715 (MAX(rsize + (int)sizeof(uint32_t), (int32_t)max_compress_len) +
716 cipher_block_size - 1) / cipher_block_size * cipher_block_size);
718 wbuf = jcr->crypto_buf; /* Encrypted, possibly compressed output here. */
722 * Send Data header to Storage daemon
723 * <file-index> <stream> <info>
725 if (!bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, stream)) {
726 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
730 Dmsg1(300, ">stored: datahdr %s\n", sd->msg);
733 * Make space at beginning of buffer for fileAddr because this
734 * same buffer will be used for writing if compression is off.
736 if (ff_pkt->flags & FO_SPARSE) {
737 rbuf += SPARSE_FADDR_SIZE;
738 rsize -= SPARSE_FADDR_SIZE;
739 #ifdef HAVE_FREEBSD_OS
741 * To read FreeBSD partitions, the read size must be
744 rsize = (rsize/512) * 512;
748 /* a RAW device read on win32 only works if the buffer is a multiple of 512 */
750 if (S_ISBLK(ff_pkt->statp.st_mode))
751 rsize = (rsize/512) * 512;
757 while ((sd->msglen=(uint32_t)bread(&ff_pkt->bfd, rbuf, rsize)) > 0) {
759 /* Check for sparse blocks */
760 if (ff_pkt->flags & FO_SPARSE) {
762 bool haveBlock = true;
763 if (sd->msglen == rsize &&
764 fileAddr+sd->msglen < (uint64_t)ff_pkt->statp.st_size ||
765 ((ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO) &&
766 (uint64_t)ff_pkt->statp.st_size == 0)) {
767 haveBlock = !is_buf_zero(rbuf, rsize);
770 ser_begin(wbuf, SPARSE_FADDR_SIZE);
771 ser_uint64(fileAddr); /* store fileAddr in begin of buffer */
773 fileAddr += sd->msglen; /* update file address */
775 continue; /* skip block of zeros */
779 jcr->ReadBytes += sd->msglen; /* count bytes read */
781 /* Uncompressed cipher input length */
782 cipher_input_len = sd->msglen;
784 /* Update checksum if requested */
786 crypto_digest_update(digest, (uint8_t *)rbuf, sd->msglen);
789 /* Update signing digest if requested */
790 if (signing_digest) {
791 crypto_digest_update(signing_digest, (uint8_t *)rbuf, sd->msglen);
795 /* Do compression if turned on */
796 if (ff_pkt->flags & FO_GZIP && jcr->pZLIB_compress_workset) {
797 Dmsg3(400, "cbuf=0x%x rbuf=0x%x len=%u\n", cbuf, rbuf, sd->msglen);
799 ((z_stream*)jcr->pZLIB_compress_workset)->next_in = (Bytef *)rbuf;
800 ((z_stream*)jcr->pZLIB_compress_workset)->avail_in = sd->msglen;
801 ((z_stream*)jcr->pZLIB_compress_workset)->next_out = (Bytef *)cbuf;
802 ((z_stream*)jcr->pZLIB_compress_workset)->avail_out = max_compress_len;
804 if ((zstat=deflate((z_stream*)jcr->pZLIB_compress_workset, Z_FINISH)) != Z_STREAM_END) {
805 Jmsg(jcr, M_FATAL, 0, _("Compression deflate error: %d\n"), zstat);
806 set_jcr_job_status(jcr, JS_ErrorTerminated);
809 compress_len = ((z_stream*)jcr->pZLIB_compress_workset)->total_out;
810 /* reset zlib stream to be able to begin from scratch again */
811 if ((zstat=deflateReset((z_stream*)jcr->pZLIB_compress_workset)) != Z_OK) {
812 Jmsg(jcr, M_FATAL, 0, _("Compression deflateReset error: %d\n"), zstat);
813 set_jcr_job_status(jcr, JS_ErrorTerminated);
817 Dmsg2(400, "compressed len=%d uncompressed len=%d\n", compress_len,
820 sd->msglen = compress_len; /* set compressed length */
821 cipher_input_len = compress_len;
825 * Note, here we prepend the current record length to the beginning
826 * of the encrypted data. This is because both sparse and compression
827 * restore handling want records returned to them with exactly the
828 * same number of bytes that were processed in the backup handling.
829 * That is, both are block filters rather than a stream. When doing
830 * compression, the compression routines may buffer data, so that for
831 * any one record compressed, when it is decompressed the same size
832 * will not be obtained. Of course, the buffered data eventually comes
833 * out in subsequent crypto_cipher_update() calls or at least
834 * when crypto_cipher_finalize() is called. Unfortunately, this
835 * "feature" of encryption enormously complicates the restore code.
837 if (ff_pkt->flags & FO_ENCRYPT) {
838 uint32_t initial_len = 0;
841 if (ff_pkt->flags & FO_SPARSE) {
842 cipher_input_len += SPARSE_FADDR_SIZE;
845 /* Encrypt the length of the input block */
846 uint8_t packet_len[sizeof(uint32_t)];
848 ser_begin(packet_len, sizeof(uint32_t));
849 ser_uint32(cipher_input_len); /* store data len in begin of buffer */
850 Dmsg1(20, "Encrypt len=%d\n", cipher_input_len);
852 if (!crypto_cipher_update(cipher_ctx, packet_len, sizeof(packet_len),
853 (u_int8_t *)jcr->crypto_buf, &initial_len)) {
854 /* Encryption failed. Shouldn't happen. */
855 Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
859 /* Encrypt the input block */
860 if (crypto_cipher_update(cipher_ctx, cipher_input, cipher_input_len,
861 (u_int8_t *)&jcr->crypto_buf[initial_len], &encrypted_len)) {
862 if ((initial_len + encrypted_len) == 0) {
863 /* No full block of data available, read more data */
866 Dmsg2(400, "encrypted len=%d unencrypted len=%d\n", encrypted_len,
868 sd->msglen = initial_len + encrypted_len; /* set encrypted length */
870 /* Encryption failed. Shouldn't happen. */
871 Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
876 /* Send the buffer to the Storage daemon */
877 if (ff_pkt->flags & FO_SPARSE) {
878 sd->msglen += SPARSE_FADDR_SIZE; /* include fileAddr in size */
880 sd->msg = wbuf; /* set correct write buffer */
881 if (!bnet_send(sd)) {
882 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
886 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
888 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
889 sd->msg = msgsave; /* restore read buffer */
891 } /* end while read file data */
893 if (sd->msglen < 0) { /* error */
895 Jmsg(jcr, M_ERROR, 0, _("Read error on file %s. ERR=%s\n"),
896 ff_pkt->fname, be.strerror(ff_pkt->bfd.berrno));
897 if (jcr->Errors++ > 1000) { /* insanity check */
898 Jmsg(jcr, M_FATAL, 0, _("Too many errors.\n"));
900 } else if (ff_pkt->flags & FO_ENCRYPT) {
902 * For encryption, we must call finalize to push out any
905 if (!crypto_cipher_finalize(cipher_ctx, (uint8_t *)jcr->crypto_buf,
907 /* Padding failed. Shouldn't happen. */
908 Jmsg(jcr, M_FATAL, 0, _("Encryption padding error\n"));
912 /* Note, on SSL pre-0.9.7, there is always some output */
913 if (encrypted_len > 0) {
914 sd->msglen = encrypted_len; /* set encrypted length */
915 sd->msg = jcr->crypto_buf; /* set correct write buffer */
916 if (!bnet_send(sd)) {
917 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
921 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
922 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
923 sd->msg = msgsave; /* restore bnet buffer */
927 if (!bnet_sig(sd, BNET_EOD)) { /* indicate end of file data */
928 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
933 /* Free the cipher context */
935 crypto_cipher_free(cipher_ctx);
940 /* Free the cipher context */
942 crypto_cipher_free(cipher_ctx);
945 sd->msg = msgsave; /* restore bnet buffer */
951 * Read and send an ACL for the last encountered file.
953 static bool read_and_send_acl(JCR *jcr, int acltype, int stream)
956 BSOCK *sd = jcr->store_bsock;
959 #ifdef FD_NO_SEND_TEST
963 len = bacl_get(jcr, acltype);
965 Jmsg1(jcr, M_WARNING, 0, _("Error reading ACL of %s\n"), jcr->last_fname);
969 return true; /* no ACL */
973 if (!bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, stream)) {
974 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
979 /* Send the buffer to the storage deamon */
980 Dmsg2(400, "Backing up ACL type 0x%2x <%s>\n", acltype, jcr->acl_text);
982 sd->msg = jcr->acl_text;
983 sd->msglen = len + 1;
984 if (!bnet_send(sd)) {
987 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
992 jcr->JobBytes += sd->msglen;
994 if (!bnet_sig(sd, BNET_EOD)) {
995 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1000 Dmsg1(200, "ACL of file: %s successfully backed up!\n", jcr->last_fname);
1005 static bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream)
1007 BSOCK *sd = jcr->store_bsock;
1008 char attribs[MAXSTRING];
1009 char attribsEx[MAXSTRING];
1012 #ifdef FD_NO_SEND_TEST
1016 /* Find what data stream we will use, then encode the attributes */
1017 if ((data_stream = select_data_stream(ff_pkt)) == STREAM_NONE) {
1018 /* This should not happen */
1019 Jmsg0(jcr, M_FATAL, 0, _("Invalid file flags, no supported data stream type.\n"));
1022 encode_stat(attribs, ff_pkt, data_stream);
1024 /* Now possibly extend the attributes */
1025 attr_stream = encode_attribsEx(jcr, attribsEx, ff_pkt);
1027 Dmsg3(300, "File %s\nattribs=%s\nattribsEx=%s\n", ff_pkt->fname, attribs, attribsEx);
1030 jcr->JobFiles++; /* increment number of files sent */
1031 ff_pkt->FileIndex = jcr->JobFiles; /* return FileIndex */
1032 pm_strcpy(jcr->last_fname, ff_pkt->fname);
1036 * Send Attributes header to Storage daemon
1037 * <file-index> <stream> <info>
1039 if (!bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, attr_stream)) {
1040 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1044 Dmsg1(300, ">stored: attrhdr %s\n", sd->msg);
1047 * Send file attributes to Storage daemon
1050 * Filename (full path)
1051 * Encoded attributes
1052 * Link name (if type==FT_LNK or FT_LNKSAVED)
1053 * Encoded extended-attributes (for Win32)
1055 * For a directory, link is the same as fname, but with trailing
1056 * slash. For a linked file, link is the link.
1058 if (ff_pkt->type == FT_LNK || ff_pkt->type == FT_LNKSAVED) {
1059 Dmsg2(300, "Link %s to %s\n", ff_pkt->fname, ff_pkt->link);
1060 stat = bnet_fsend(sd, "%ld %d %s%c%s%c%s%c%s%c", jcr->JobFiles,
1061 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, ff_pkt->link, 0,
1063 } else if (ff_pkt->type == FT_DIREND) {
1064 /* Here link is the canonical filename (i.e. with trailing slash) */
1065 stat = bnet_fsend(sd, "%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
1066 ff_pkt->type, ff_pkt->link, 0, attribs, 0, 0, attribsEx, 0);
1068 stat = bnet_fsend(sd, "%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
1069 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, 0, attribsEx, 0);
1072 Dmsg2(300, ">stored: attr len=%d: %s\n", sd->msglen, sd->msg);
1074 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1078 bnet_sig(sd, BNET_EOD); /* indicate end of attributes data */