2 Bacula® - The Network Backup Solution
4 Copyright (C) 2000-2010 Free Software Foundation Europe e.V.
6 The main author of Bacula is Kern Sibbald, with contributions from
7 many others, a complete list can be found in the file AUTHORS.
8 This program is Free Software; you can redistribute it and/or
9 modify it under the terms of version two of the GNU General Public
10 License as published by the Free Software Foundation and included
13 This program is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
23 Bacula® is a registered trademark of Kern Sibbald.
24 The licensor of Bacula is the Free Software Foundation Europe
25 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
26 Switzerland, email:ftf@fsfeurope.org.
29 * Bacula File Daemon backup.c send file attributes and data
30 * to the Storage daemon.
32 * Kern Sibbald, March MM
40 const bool have_darwin_os = true;
42 const bool have_darwin_os = false;
46 const bool have_acl = true;
48 const bool have_acl = false;
51 #if defined(HAVE_XATTR)
52 const bool have_xattr = true;
54 const bool have_xattr = false;
57 /* Forward referenced functions */
58 int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level);
59 static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, DIGEST *signature_digest);
60 bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream);
61 static bool crypto_session_start(JCR *jcr);
62 static void crypto_session_end(JCR *jcr);
63 static bool crypto_session_send(JCR *jcr, BSOCK *sd);
66 * Find all the requested files and send them
67 * to the Storage daemon.
69 * Note, we normally carry on a one-way
70 * conversation from this point on with the SD, simply blasting
71 * data to him. To properly know what is going on, we
72 * also run a "heartbeat" monitor which reads the socket and
73 * reacts accordingly (at the moment it has nothing to do
74 * except echo the heartbeat to the Director).
77 bool blast_data_to_storage_daemon(JCR *jcr, char *addr)
81 // TODO landonf: Allow user to specify encryption algorithm
83 sd = jcr->store_bsock;
85 set_jcr_job_status(jcr, JS_Running);
87 Dmsg1(300, "bfiled: opened data connection %d to stored\n", sd->m_fd);
90 CLIENT *client = (CLIENT *)GetNextRes(R_CLIENT, NULL);
94 buf_size = client->max_network_buffer_size;
96 buf_size = 0; /* use default */
98 if (!sd->set_buffer_size(buf_size, BNET_SETBUF_WRITE)) {
99 set_jcr_job_status(jcr, JS_ErrorTerminated);
100 Jmsg(jcr, M_FATAL, 0, _("Cannot set buffer size FD->SD.\n"));
104 jcr->buf_size = sd->msglen;
106 * Adjust for compression so that output buffer is
107 * 12 bytes + 0.1% larger than input buffer plus 18 bytes.
108 * This gives a bit extra plus room for the sparse addr if any.
109 * Note, we adjust the read size to be smaller so that the
110 * same output buffer can be used without growing it.
112 * The zlib compression workset is initialized here to minimize
113 * the "per file" load. The jcr member is only set, if the init
116 jcr->compress_buf_size = jcr->buf_size + ((jcr->buf_size+999) / 1000) + 30;
117 jcr->compress_buf = get_memory(jcr->compress_buf_size);
120 z_stream *pZlibStream = (z_stream*)malloc(sizeof(z_stream));
122 pZlibStream->zalloc = Z_NULL;
123 pZlibStream->zfree = Z_NULL;
124 pZlibStream->opaque = Z_NULL;
125 pZlibStream->state = Z_NULL;
127 if (deflateInit(pZlibStream, Z_DEFAULT_COMPRESSION) == Z_OK) {
128 jcr->pZLIB_compress_workset = pZlibStream;
135 if (!crypto_session_start(jcr)) {
139 set_find_options((FF_PKT *)jcr->ff, jcr->incremental, jcr->mtime);
141 /** in accurate mode, we overwrite the find_one check function */
143 set_find_changed_function((FF_PKT *)jcr->ff, accurate_check_file);
146 start_heartbeat_monitor(jcr);
149 jcr->acl_data = (acl_data_t *)malloc(sizeof(acl_data_t));
150 memset((caddr_t)jcr->acl_data, 0, sizeof(acl_data_t));
151 jcr->acl_data->content = get_pool_memory(PM_MESSAGE);
155 jcr->xattr_data = (xattr_data_t *)malloc(sizeof(xattr_data_t));
156 memset((caddr_t)jcr->xattr_data, 0, sizeof(xattr_data_t));
157 jcr->xattr_data->content = get_pool_memory(PM_MESSAGE);
160 /** Subroutine save_file() is called for each file */
161 if (!find_files(jcr, (FF_PKT *)jcr->ff, save_file, plugin_save)) {
162 ok = false; /* error */
163 set_jcr_job_status(jcr, JS_ErrorTerminated);
166 if (have_acl && jcr->acl_data->nr_errors > 0) {
167 Jmsg(jcr, M_ERROR, 0, _("Encountered %ld acl errors while doing backup\n"),
168 jcr->acl_data->nr_errors);
170 if (have_xattr && jcr->xattr_data->nr_errors > 0) {
171 Jmsg(jcr, M_ERROR, 0, _("Encountered %ld xattr errors while doing backup\n"),
172 jcr->xattr_data->nr_errors);
175 accurate_finish(jcr); /* send deleted or base file list to SD */
177 stop_heartbeat_monitor(jcr);
179 sd->signal(BNET_EOD); /* end of sending data */
181 if (have_acl && jcr->acl_data) {
182 free_pool_memory(jcr->acl_data->content);
184 jcr->acl_data = NULL;
186 if (have_xattr && jcr->xattr_data) {
187 free_pool_memory(jcr->xattr_data->content);
188 free(jcr->xattr_data);
189 jcr->xattr_data = NULL;
195 if (jcr->compress_buf) {
196 free_pool_memory(jcr->compress_buf);
197 jcr->compress_buf = NULL;
199 if (jcr->pZLIB_compress_workset) {
200 /* Free the zlib stream */
202 deflateEnd((z_stream *)jcr->pZLIB_compress_workset);
204 free (jcr->pZLIB_compress_workset);
205 jcr->pZLIB_compress_workset = NULL;
207 crypto_session_end(jcr);
210 Dmsg1(100, "end blast_data ok=%d\n", ok);
214 static bool crypto_session_start(JCR *jcr)
216 crypto_cipher_t cipher = CRYPTO_CIPHER_AES_128_CBC;
219 * Create encryption session data and a cached, DER-encoded session data
220 * structure. We use a single session key for each backup, so we'll encode
221 * the session data only once.
223 if (jcr->crypto.pki_encrypt) {
226 /** Create per-job session encryption context */
227 jcr->crypto.pki_session = crypto_session_new(cipher, jcr->crypto.pki_recipients);
229 /** Get the session data size */
230 if (!crypto_session_encode(jcr->crypto.pki_session, (uint8_t *)0, &size)) {
231 Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n"));
235 /** Allocate buffer */
236 jcr->crypto.pki_session_encoded = get_memory(size);
238 /** Encode session data */
239 if (!crypto_session_encode(jcr->crypto.pki_session, (uint8_t *)jcr->crypto.pki_session_encoded, &size)) {
240 Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n"));
244 /** ... and store the encoded size */
245 jcr->crypto.pki_session_encoded_size = size;
247 /** Allocate the encryption/decryption buffer */
248 jcr->crypto.crypto_buf = get_memory(CRYPTO_CIPHER_MAX_BLOCK_SIZE);
253 static void crypto_session_end(JCR *jcr)
255 if (jcr->crypto.crypto_buf) {
256 free_pool_memory(jcr->crypto.crypto_buf);
257 jcr->crypto.crypto_buf = NULL;
259 if (jcr->crypto.pki_session) {
260 crypto_session_free(jcr->crypto.pki_session);
262 if (jcr->crypto.pki_session_encoded) {
263 free_pool_memory(jcr->crypto.pki_session_encoded);
264 jcr->crypto.pki_session_encoded = NULL;
268 static bool crypto_session_send(JCR *jcr, BSOCK *sd)
272 /** Send our header */
273 Dmsg2(100, "Send hdr fi=%ld stream=%d\n", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA);
274 sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA);
277 sd->msg = jcr->crypto.pki_session_encoded;
278 sd->msglen = jcr->crypto.pki_session_encoded_size;
279 jcr->JobBytes += sd->msglen;
281 Dmsg1(100, "Send data len=%d\n", sd->msglen);
284 sd->signal(BNET_EOD);
290 * Called here by find() for each file included.
291 * This is a callback. The original is find_files() above.
293 * Send the file and its data to the Storage daemon.
297 * -1 to ignore file/directory (not used here)
299 int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level)
301 bool do_read = false;
302 int stat, data_stream;
304 DIGEST *digest = NULL;
305 DIGEST *signing_digest = NULL;
306 int digest_stream = STREAM_NONE;
307 SIGNATURE *sig = NULL;
308 bool has_file_data = false;
309 // TODO landonf: Allow the user to specify the digest algorithm
311 crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA256;
313 crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA1;
315 BSOCK *sd = jcr->store_bsock;
317 if (jcr->is_job_canceled()) {
321 jcr->num_files_examined++; /* bump total file count */
323 switch (ff_pkt->type) {
324 case FT_LNKSAVED: /* Hard linked, file already saved */
325 Dmsg2(130, "FT_LNKSAVED hard link: %s => %s\n", ff_pkt->fname, ff_pkt->link);
328 Dmsg1(130, "FT_REGE saving: %s\n", ff_pkt->fname);
329 has_file_data = true;
332 Dmsg1(130, "FT_REG saving: %s\n", ff_pkt->fname);
333 has_file_data = true;
336 Dmsg2(130, "FT_LNK saving: %s -> %s\n", ff_pkt->fname, ff_pkt->link);
338 case FT_RESTORE_FIRST:
339 Dmsg1(00, "FT_RESTORE_FIRST saving: %s\n", ff_pkt->fname);
342 jcr->num_files_examined--; /* correct file count */
343 return 1; /* not used */
345 Jmsg(jcr, M_INFO, 1, _(" Recursion turned off. Will not descend from %s into %s\n"),
346 ff_pkt->top_fname, ff_pkt->fname);
347 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
350 /* Suppress message for /dev filesystems */
351 if (!is_in_fileset(ff_pkt)) {
352 Jmsg(jcr, M_INFO, 1, _(" %s is a different filesystem. Will not descend from %s into %s\n"),
353 ff_pkt->fname, ff_pkt->top_fname, ff_pkt->fname);
355 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
358 Jmsg(jcr, M_INFO, 1, _(" Disallowed filesystem. Will not descend from %s into %s\n"),
359 ff_pkt->top_fname, ff_pkt->fname);
360 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
363 Jmsg(jcr, M_INFO, 1, _(" Disallowed drive type. Will not descend into %s\n"),
368 Dmsg1(130, "FT_DIREND: %s\n", ff_pkt->link);
371 Dmsg1(130, "FT_SPEC saving: %s\n", ff_pkt->fname);
372 if (S_ISSOCK(ff_pkt->statp.st_mode)) {
373 Jmsg(jcr, M_SKIPPED, 1, _(" Socket file skipped: %s\n"), ff_pkt->fname);
378 Dmsg1(130, "FT_RAW saving: %s\n", ff_pkt->fname);
379 has_file_data = true;
382 Dmsg1(130, "FT_FIFO saving: %s\n", ff_pkt->fname);
386 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not access \"%s\": ERR=%s\n"), ff_pkt->fname,
387 be.bstrerror(ff_pkt->ff_errno));
393 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not follow link \"%s\": ERR=%s\n"),
394 ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
400 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not stat \"%s\": ERR=%s\n"), ff_pkt->fname,
401 be.bstrerror(ff_pkt->ff_errno));
407 Jmsg(jcr, M_SKIPPED, 1, _(" Unchanged file skipped: %s\n"), ff_pkt->fname);
410 Jmsg(jcr, M_NOTSAVED, 0, _(" Archive file not saved: %s\n"), ff_pkt->fname);
414 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not open directory \"%s\": ERR=%s\n"),
415 ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
420 Jmsg(jcr, M_NOTSAVED, 0, _(" Unknown file type %d; not saved: %s\n"),
421 ff_pkt->type, ff_pkt->fname);
426 Dmsg1(130, "bfiled: sending %s to stored\n", ff_pkt->fname);
428 /** Digests and encryption are only useful if there's file data */
431 * Setup for digest handling. If this fails, the digest will be set to NULL
432 * and not used. Note, the digest (file hash) can be any one of the four
435 * The signing digest is a single algorithm depending on
436 * whether or not we have SHA2.
437 * ****FIXME**** the signing algoritm should really be
438 * determined a different way!!!!!! What happens if
439 * sha2 was available during backup but not restore?
441 if (ff_pkt->flags & FO_MD5) {
442 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_MD5);
443 digest_stream = STREAM_MD5_DIGEST;
445 } else if (ff_pkt->flags & FO_SHA1) {
446 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA1);
447 digest_stream = STREAM_SHA1_DIGEST;
449 } else if (ff_pkt->flags & FO_SHA256) {
450 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA256);
451 digest_stream = STREAM_SHA256_DIGEST;
453 } else if (ff_pkt->flags & FO_SHA512) {
454 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA512);
455 digest_stream = STREAM_SHA512_DIGEST;
458 /** Did digest initialization fail? */
459 if (digest_stream != STREAM_NONE && digest == NULL) {
460 Jmsg(jcr, M_WARNING, 0, _("%s digest initialization failed\n"),
461 stream_to_ascii(digest_stream));
465 * Set up signature digest handling. If this fails, the signature digest
466 * will be set to NULL and not used.
468 /* TODO landonf: We should really only calculate the digest once, for
469 * both verification and signing.
471 if (jcr->crypto.pki_sign) {
472 signing_digest = crypto_digest_new(jcr, signing_algorithm);
474 /** Full-stop if a failure occurred initializing the signature digest */
475 if (signing_digest == NULL) {
476 Jmsg(jcr, M_NOTSAVED, 0, _("%s signature digest initialization failed\n"),
477 stream_to_ascii(signing_algorithm));
483 /** Enable encryption */
484 if (jcr->crypto.pki_encrypt) {
485 ff_pkt->flags |= FO_ENCRYPT;
489 /** Initialize the file descriptor we use for data and other streams. */
491 if (ff_pkt->flags & FO_PORTABLE) {
492 set_portable_backup(&ff_pkt->bfd); /* disable Win32 BackupRead() */
494 if (ff_pkt->cmd_plugin) {
495 /* Tell bfile that it needs to call plugin */
496 if (!set_cmd_plugin(&ff_pkt->bfd, jcr)) {
499 send_plugin_name(jcr, sd, true); /* signal start of plugin data */
502 /** Send attributes -- must be done after binit() */
503 if (!encode_and_send_attributes(jcr, ff_pkt, data_stream)) {
507 /** Set up the encryption context and send the session data to the SD */
508 if (has_file_data && jcr->crypto.pki_encrypt) {
509 if (!crypto_session_send(jcr, sd)) {
515 * Open any file with data that we intend to save, then save it.
517 * Note, if is_win32_backup, we must open the Directory so that
518 * the BackupRead will save its permissions and ownership streams.
520 if (ff_pkt->type != FT_LNKSAVED && S_ISREG(ff_pkt->statp.st_mode)) {
522 do_read = !is_portable_backup(&ff_pkt->bfd) || ff_pkt->statp.st_size > 0;
524 do_read = ff_pkt->statp.st_size > 0;
526 } else if (ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO ||
527 ff_pkt->type == FT_REPARSE ||
528 (!is_portable_backup(&ff_pkt->bfd) && ff_pkt->type == FT_DIREND)) {
532 if (ff_pkt->cmd_plugin && ff_pkt->type != FT_RESTORE_FIRST) {
536 Dmsg2(000, "type=%d do_read=%d\n", ff_pkt->type, do_read);
540 if (ff_pkt->type == FT_FIFO) {
541 tid = start_thread_timer(jcr, pthread_self(), 60);
545 int noatime = ff_pkt->flags & FO_NOATIME ? O_NOATIME : 0;
546 ff_pkt->bfd.reparse_point = ff_pkt->type == FT_REPARSE;
547 if (bopen(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY | noatime, 0) < 0) {
548 ff_pkt->ff_errno = errno;
550 Jmsg(jcr, M_NOTSAVED, 0, _(" Cannot open \"%s\": ERR=%s.\n"), ff_pkt->fname,
554 stop_thread_timer(tid);
560 stop_thread_timer(tid);
564 stat = send_data(jcr, data_stream, ff_pkt, digest, signing_digest);
566 if (ff_pkt->flags & FO_CHKCHANGES) {
567 has_file_changed(jcr, ff_pkt);
570 bclose(&ff_pkt->bfd);
577 if (have_darwin_os) {
578 /** Regular files can have resource forks and Finder Info */
579 if (ff_pkt->type != FT_LNKSAVED && (S_ISREG(ff_pkt->statp.st_mode) &&
580 ff_pkt->flags & FO_HFSPLUS)) {
581 if (ff_pkt->hfsinfo.rsrclength > 0) {
584 if (!bopen_rsrc(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0) < 0) {
585 ff_pkt->ff_errno = errno;
587 Jmsg(jcr, M_NOTSAVED, -1, _(" Cannot open resource fork for \"%s\": ERR=%s.\n"),
588 ff_pkt->fname, be.bstrerror());
590 if (is_bopen(&ff_pkt->bfd)) {
591 bclose(&ff_pkt->bfd);
595 flags = ff_pkt->flags;
596 ff_pkt->flags &= ~(FO_GZIP|FO_SPARSE);
597 if (flags & FO_ENCRYPT) {
598 rsrc_stream = STREAM_ENCRYPTED_MACOS_FORK_DATA;
600 rsrc_stream = STREAM_MACOS_FORK_DATA;
602 stat = send_data(jcr, rsrc_stream, ff_pkt, digest, signing_digest);
603 ff_pkt->flags = flags;
604 bclose(&ff_pkt->bfd);
610 Dmsg1(300, "Saving Finder Info for \"%s\"\n", ff_pkt->fname);
611 sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_HFSPLUS_ATTRIBUTES);
612 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
613 pm_memcpy(sd->msg, ff_pkt->hfsinfo.fndrinfo, 32);
616 crypto_digest_update(digest, (uint8_t *)sd->msg, sd->msglen);
618 if (signing_digest) {
619 crypto_digest_update(signing_digest, (uint8_t *)sd->msg, sd->msglen);
622 sd->signal(BNET_EOD);
627 * Save ACLs when requested and available for anything not being a symlink and not being a plugin.
630 if (ff_pkt->flags & FO_ACL && ff_pkt->type != FT_LNK && !ff_pkt->cmd_plugin) {
631 switch (build_acl_streams(jcr, ff_pkt)) {
632 case bacl_exit_fatal:
634 case bacl_exit_error:
636 * Non-fatal errors, count them and when the number is under ACL_REPORT_ERR_MAX_PER_JOB
637 * print the error message set by the lower level routine in jcr->errmsg.
639 if (jcr->acl_data->nr_errors < ACL_REPORT_ERR_MAX_PER_JOB) {
640 Jmsg(jcr, M_ERROR, 0, "%s", jcr->errmsg);
642 jcr->acl_data->nr_errors++;
651 * Save Extended Attributes when requested and available for all files not being a plugin.
654 if (ff_pkt->flags & FO_XATTR && !ff_pkt->cmd_plugin) {
655 switch (build_xattr_streams(jcr, ff_pkt)) {
656 case bxattr_exit_fatal:
658 case bxattr_exit_error:
660 * Non-fatal errors, count them and when the number is under XATTR_REPORT_ERR_MAX_PER_JOB
661 * print the error message set by the lower level routine in jcr->errmsg.
663 if (jcr->xattr_data->nr_errors < XATTR_REPORT_ERR_MAX_PER_JOB) {
664 Jmsg(jcr, M_ERROR, 0, "%s", jcr->errmsg);
666 jcr->xattr_data->nr_errors++;
674 /** Terminate the signing digest and send it to the Storage daemon */
675 if (signing_digest) {
678 if ((sig = crypto_sign_new(jcr)) == NULL) {
679 Jmsg(jcr, M_FATAL, 0, _("Failed to allocate memory for crypto signature.\n"));
683 if (!crypto_sign_add_signer(sig, signing_digest, jcr->crypto.pki_keypair)) {
684 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
688 /** Get signature size */
689 if (!crypto_sign_encode(sig, NULL, &size)) {
690 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
694 /** Grow the bsock buffer to fit our message if necessary */
695 if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
696 sd->msg = realloc_pool_memory(sd->msg, size);
699 /** Send our header */
700 sd->fsend("%ld %ld 0", jcr->JobFiles, STREAM_SIGNED_DIGEST);
701 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
703 /** Encode signature data */
704 if (!crypto_sign_encode(sig, (uint8_t *)sd->msg, &size)) {
705 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
711 sd->signal(BNET_EOD); /* end of checksum */
714 /** Terminate any digest and send it to Storage daemon */
718 sd->fsend("%ld %d 0", jcr->JobFiles, digest_stream);
719 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
721 size = CRYPTO_DIGEST_MAX_SIZE;
723 /** Grow the bsock buffer to fit our message if necessary */
724 if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
725 sd->msg = realloc_pool_memory(sd->msg, size);
728 if (!crypto_digest_finalize(digest, (uint8_t *)sd->msg, &size)) {
729 Jmsg(jcr, M_FATAL, 0, _("An error occurred finalizing signing the stream.\n"));
735 sd->signal(BNET_EOD); /* end of checksum */
737 if (ff_pkt->cmd_plugin) {
738 send_plugin_name(jcr, sd, false); /* signal end of plugin data */
742 rtnstat = 1; /* good return */
746 crypto_digest_free(digest);
748 if (signing_digest) {
749 crypto_digest_free(signing_digest);
752 crypto_sign_free(sig);
758 * Send data read from an already open file descriptor.
760 * We return 1 on sucess and 0 on errors.
763 * We use ff_pkt->statp.st_size when FO_SPARSE to know when to stop
765 * Currently this is not a problem as the only other stream, resource forks,
766 * are not handled as sparse files.
768 static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest,
769 DIGEST *signing_digest)
771 BSOCK *sd = jcr->store_bsock;
772 uint64_t fileAddr = 0; /* file address */
774 int32_t rsize = jcr->buf_size; /* read buffer size */
776 CIPHER_CONTEXT *cipher_ctx = NULL; /* Quell bogus uninitialized warnings */
777 const uint8_t *cipher_input;
778 uint32_t cipher_input_len;
779 uint32_t cipher_block_size;
780 uint32_t encrypted_len;
781 #ifdef FD_NO_SEND_TEST
786 rbuf = sd->msg; /* read buffer */
787 wbuf = sd->msg; /* write buffer */
788 cipher_input = (uint8_t *)rbuf; /* encrypt uncompressed data */
790 Dmsg1(300, "Saving data, type=%d\n", ff_pkt->type);
793 uLong compress_len = 0;
794 uLong max_compress_len = 0;
795 const Bytef *cbuf = NULL;
798 if (ff_pkt->flags & FO_GZIP) {
799 if (ff_pkt->flags & FO_SPARSE) {
800 cbuf = (Bytef *)jcr->compress_buf + SPARSE_FADDR_SIZE;
801 max_compress_len = jcr->compress_buf_size - SPARSE_FADDR_SIZE;
803 cbuf = (Bytef *)jcr->compress_buf;
804 max_compress_len = jcr->compress_buf_size; /* set max length */
806 wbuf = jcr->compress_buf; /* compressed output here */
807 cipher_input = (uint8_t *)jcr->compress_buf; /* encrypt compressed data */
810 * Only change zlib parameters if there is no pending operation.
811 * This should never happen as deflatereset is called after each
815 if (((z_stream*)jcr->pZLIB_compress_workset)->total_in == 0) {
816 /** set gzip compression level - must be done per file */
817 if ((zstat=deflateParams((z_stream*)jcr->pZLIB_compress_workset,
818 ff_pkt->GZIP_level, Z_DEFAULT_STRATEGY)) != Z_OK) {
819 Jmsg(jcr, M_FATAL, 0, _("Compression deflateParams error: %d\n"), zstat);
820 set_jcr_job_status(jcr, JS_ErrorTerminated);
826 const uint32_t max_compress_len = 0;
829 if (ff_pkt->flags & FO_ENCRYPT) {
830 if (ff_pkt->flags & FO_SPARSE) {
831 Jmsg0(jcr, M_FATAL, 0, _("Encrypting sparse data not supported.\n"));
834 /** Allocate the cipher context */
835 if ((cipher_ctx = crypto_cipher_new(jcr->crypto.pki_session, true,
836 &cipher_block_size)) == NULL) {
837 /* Shouldn't happen! */
838 Jmsg0(jcr, M_FATAL, 0, _("Failed to initialize encryption context.\n"));
843 * Grow the crypto buffer, if necessary.
844 * crypto_cipher_update() will buffer up to (cipher_block_size - 1).
845 * We grow crypto_buf to the maximum number of blocks that
846 * could be returned for the given read buffer size.
847 * (Using the larger of either rsize or max_compress_len)
849 jcr->crypto.crypto_buf = check_pool_memory_size(jcr->crypto.crypto_buf,
850 (MAX(rsize + (int)sizeof(uint32_t), (int32_t)max_compress_len) +
851 cipher_block_size - 1) / cipher_block_size * cipher_block_size);
853 wbuf = jcr->crypto.crypto_buf; /* Encrypted, possibly compressed output here. */
857 * Send Data header to Storage daemon
858 * <file-index> <stream> <info>
860 if (!sd->fsend("%ld %d 0", jcr->JobFiles, stream)) {
861 if (!jcr->is_job_canceled()) {
862 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
867 Dmsg1(300, ">stored: datahdr %s\n", sd->msg);
870 * Make space at beginning of buffer for fileAddr because this
871 * same buffer will be used for writing if compression is off.
873 if (ff_pkt->flags & FO_SPARSE) {
874 rbuf += SPARSE_FADDR_SIZE;
875 rsize -= SPARSE_FADDR_SIZE;
876 #ifdef HAVE_FREEBSD_OS
878 * To read FreeBSD partitions, the read size must be
881 rsize = (rsize/512) * 512;
885 /** a RAW device read on win32 only works if the buffer is a multiple of 512 */
887 if (S_ISBLK(ff_pkt->statp.st_mode))
888 rsize = (rsize/512) * 512;
894 while ((sd->msglen=(uint32_t)bread(&ff_pkt->bfd, rbuf, rsize)) > 0) {
896 /** Check for sparse blocks */
897 if (ff_pkt->flags & FO_SPARSE) {
899 bool allZeros = false;
900 if ((sd->msglen == rsize &&
901 fileAddr+sd->msglen < (uint64_t)ff_pkt->statp.st_size) ||
902 ((ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO) &&
903 (uint64_t)ff_pkt->statp.st_size == 0)) {
904 allZeros = is_buf_zero(rbuf, rsize);
907 /** Put file address as first data in buffer */
908 ser_begin(wbuf, SPARSE_FADDR_SIZE);
909 ser_uint64(fileAddr); /* store fileAddr in begin of buffer */
911 fileAddr += sd->msglen; /* update file address */
912 /** Skip block of all zeros */
914 continue; /* skip block of zeros */
918 jcr->ReadBytes += sd->msglen; /* count bytes read */
920 /** Uncompressed cipher input length */
921 cipher_input_len = sd->msglen;
923 /** Update checksum if requested */
925 crypto_digest_update(digest, (uint8_t *)rbuf, sd->msglen);
928 /** Update signing digest if requested */
929 if (signing_digest) {
930 crypto_digest_update(signing_digest, (uint8_t *)rbuf, sd->msglen);
934 /** Do compression if turned on */
935 if (ff_pkt->flags & FO_GZIP && jcr->pZLIB_compress_workset) {
936 Dmsg3(400, "cbuf=0x%x rbuf=0x%x len=%u\n", cbuf, rbuf, sd->msglen);
938 ((z_stream*)jcr->pZLIB_compress_workset)->next_in = (Bytef *)rbuf;
939 ((z_stream*)jcr->pZLIB_compress_workset)->avail_in = sd->msglen;
940 ((z_stream*)jcr->pZLIB_compress_workset)->next_out = (Bytef *)cbuf;
941 ((z_stream*)jcr->pZLIB_compress_workset)->avail_out = max_compress_len;
943 if ((zstat=deflate((z_stream*)jcr->pZLIB_compress_workset, Z_FINISH)) != Z_STREAM_END) {
944 Jmsg(jcr, M_FATAL, 0, _("Compression deflate error: %d\n"), zstat);
945 set_jcr_job_status(jcr, JS_ErrorTerminated);
948 compress_len = ((z_stream*)jcr->pZLIB_compress_workset)->total_out;
949 /** reset zlib stream to be able to begin from scratch again */
950 if ((zstat=deflateReset((z_stream*)jcr->pZLIB_compress_workset)) != Z_OK) {
951 Jmsg(jcr, M_FATAL, 0, _("Compression deflateReset error: %d\n"), zstat);
952 set_jcr_job_status(jcr, JS_ErrorTerminated);
956 Dmsg2(400, "compressed len=%d uncompressed len=%d\n", compress_len,
959 sd->msglen = compress_len; /* set compressed length */
960 cipher_input_len = compress_len;
964 * Note, here we prepend the current record length to the beginning
965 * of the encrypted data. This is because both sparse and compression
966 * restore handling want records returned to them with exactly the
967 * same number of bytes that were processed in the backup handling.
968 * That is, both are block filters rather than a stream. When doing
969 * compression, the compression routines may buffer data, so that for
970 * any one record compressed, when it is decompressed the same size
971 * will not be obtained. Of course, the buffered data eventually comes
972 * out in subsequent crypto_cipher_update() calls or at least
973 * when crypto_cipher_finalize() is called. Unfortunately, this
974 * "feature" of encryption enormously complicates the restore code.
976 if (ff_pkt->flags & FO_ENCRYPT) {
977 uint32_t initial_len = 0;
980 if (ff_pkt->flags & FO_SPARSE) {
981 cipher_input_len += SPARSE_FADDR_SIZE;
984 /** Encrypt the length of the input block */
985 uint8_t packet_len[sizeof(uint32_t)];
987 ser_begin(packet_len, sizeof(uint32_t));
988 ser_uint32(cipher_input_len); /* store data len in begin of buffer */
989 Dmsg1(20, "Encrypt len=%d\n", cipher_input_len);
991 if (!crypto_cipher_update(cipher_ctx, packet_len, sizeof(packet_len),
992 (uint8_t *)jcr->crypto.crypto_buf, &initial_len)) {
993 /** Encryption failed. Shouldn't happen. */
994 Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
998 /** Encrypt the input block */
999 if (crypto_cipher_update(cipher_ctx, cipher_input, cipher_input_len,
1000 (uint8_t *)&jcr->crypto.crypto_buf[initial_len], &encrypted_len)) {
1001 if ((initial_len + encrypted_len) == 0) {
1002 /** No full block of data available, read more data */
1005 Dmsg2(400, "encrypted len=%d unencrypted len=%d\n", encrypted_len,
1007 sd->msglen = initial_len + encrypted_len; /* set encrypted length */
1009 /** Encryption failed. Shouldn't happen. */
1010 Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
1015 /* Send the buffer to the Storage daemon */
1016 if (ff_pkt->flags & FO_SPARSE) {
1017 sd->msglen += SPARSE_FADDR_SIZE; /* include fileAddr in size */
1019 sd->msg = wbuf; /* set correct write buffer */
1021 if (!jcr->is_job_canceled()) {
1022 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1027 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
1029 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
1030 sd->msg = msgsave; /* restore read buffer */
1032 } /* end while read file data */
1034 if (sd->msglen < 0) { /* error */
1036 Jmsg(jcr, M_ERROR, 0, _("Read error on file %s. ERR=%s\n"),
1037 ff_pkt->fname, be.bstrerror(ff_pkt->bfd.berrno));
1038 if (jcr->JobErrors++ > 1000) { /* insanity check */
1039 Jmsg(jcr, M_FATAL, 0, _("Too many errors. JobErrors=%d.\n"), jcr->JobErrors);
1041 } else if (ff_pkt->flags & FO_ENCRYPT) {
1043 * For encryption, we must call finalize to push out any
1046 if (!crypto_cipher_finalize(cipher_ctx, (uint8_t *)jcr->crypto.crypto_buf,
1048 /* Padding failed. Shouldn't happen. */
1049 Jmsg(jcr, M_FATAL, 0, _("Encryption padding error\n"));
1053 /** Note, on SSL pre-0.9.7, there is always some output */
1054 if (encrypted_len > 0) {
1055 sd->msglen = encrypted_len; /* set encrypted length */
1056 sd->msg = jcr->crypto.crypto_buf; /* set correct write buffer */
1058 if (!jcr->is_job_canceled()) {
1059 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1064 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
1065 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
1066 sd->msg = msgsave; /* restore bnet buffer */
1070 if (!sd->signal(BNET_EOD)) { /* indicate end of file data */
1071 if (!jcr->is_job_canceled()) {
1072 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1078 /** Free the cipher context */
1080 crypto_cipher_free(cipher_ctx);
1085 /** Free the cipher context */
1087 crypto_cipher_free(cipher_ctx);
1090 sd->msg = msgsave; /* restore bnet buffer */
1095 bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream)
1097 BSOCK *sd = jcr->store_bsock;
1098 char attribs[MAXSTRING];
1099 char attribsExBuf[MAXSTRING];
1103 #ifdef FD_NO_SEND_TEST
1107 Dmsg1(300, "encode_and_send_attrs fname=%s\n", ff_pkt->fname);
1108 /** Find what data stream we will use, then encode the attributes */
1109 if ((data_stream = select_data_stream(ff_pkt)) == STREAM_NONE) {
1110 /* This should not happen */
1111 Jmsg0(jcr, M_FATAL, 0, _("Invalid file flags, no supported data stream type.\n"));
1114 encode_stat(attribs, &ff_pkt->statp, ff_pkt->LinkFI, data_stream);
1116 /** Now possibly extend the attributes */
1117 if (ff_pkt->type == FT_RESTORE_FIRST) {
1119 * For restore objects, we return the object in the extended
1122 attribsEx = ff_pkt->object;
1123 attr_stream = STREAM_UNIX_ATTRIBUTES_EX;
1125 attribsEx = attribsExBuf;
1126 attr_stream = encode_attribsEx(jcr, attribsEx, ff_pkt);
1129 Dmsg3(300, "File %s\nattribs=%s\nattribsEx=%s\n", ff_pkt->fname, attribs, attribsEx);
1132 jcr->JobFiles++; /* increment number of files sent */
1133 ff_pkt->FileIndex = jcr->JobFiles; /* return FileIndex */
1134 pm_strcpy(jcr->last_fname, ff_pkt->fname);
1138 * Send Attributes header to Storage daemon
1139 * <file-index> <stream> <info>
1141 if (!sd->fsend("%ld %d 0", jcr->JobFiles, attr_stream)) {
1142 if (!jcr->is_job_canceled()) {
1143 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1148 Dmsg1(300, ">stored: attrhdr %s\n", sd->msg);
1151 * Send file attributes to Storage daemon
1154 * Filename (full path)
1155 * Encoded attributes
1156 * Link name (if type==FT_LNK or FT_LNKSAVED)
1157 * Encoded extended-attributes (for Win32)
1159 * For a directory, link is the same as fname, but with trailing
1160 * slash. For a linked file, link is the link.
1162 if (ff_pkt->type != FT_DELETED) { /* already stripped */
1165 switch (ff_pkt->type) {
1168 Dmsg2(300, "Link %s to %s\n", ff_pkt->fname, ff_pkt->link);
1169 stat = sd->fsend("%ld %d %s%c%s%c%s%c%s%c", jcr->JobFiles,
1170 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, ff_pkt->link, 0,
1175 /* Here link is the canonical filename (i.e. with trailing slash) */
1176 stat = sd->fsend("%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
1177 ff_pkt->type, ff_pkt->link, 0, attribs, 0, 0, attribsEx, 0);
1179 case FT_RESTORE_FIRST:
1181 * Note, we edit everything as we do for the default case, but the
1182 * object is tacked on to the end in place of the extended attributes,
1183 * but we do a memcpy so that the object can be a binary object.
1185 Dmsg6(000, "Type=%d DataStream=%d attrStream=%d File=%s\nattribs=%s\nattribsEx=%s",
1186 ff_pkt->type, data_stream, STREAM_UNIX_ATTRIBUTES_EX,
1187 ff_pkt->fname, attribs, ff_pkt->object);
1188 sd->msglen = Mmsg(sd->msg, "%ld %d %s%c%s%c%c",
1189 jcr->JobFiles, ff_pkt->type, ff_pkt->fname, 0, attribs, 0, 0);
1190 sd->msg = check_pool_memory_size(sd->msg, sd->msglen + ff_pkt->object_len + 1);
1191 memcpy(sd->msg + sd->msglen, ff_pkt->object, ff_pkt->object_len);
1192 sd->msglen += ff_pkt->object_len;
1195 stat = sd->fsend("%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
1196 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, 0, attribsEx, 0);
1199 if (ff_pkt->type != FT_DELETED) {
1200 unstrip_path(ff_pkt);
1203 Dmsg2(300, ">stored: attr len=%d: %s\n", sd->msglen, sd->msg);
1204 if (!stat && !jcr->is_job_canceled()) {
1205 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1208 sd->signal(BNET_EOD); /* indicate end of attributes data */
1213 * Do in place strip of path
1215 static bool do_strip(int count, char *in)
1221 /** Copy to first path separator -- Win32 might have c: ... */
1222 while (*in && !IsPathSeparator(*in)) {
1226 numsep++; /* one separator seen */
1227 for (stripped=0; stripped<count && *in; stripped++) {
1228 while (*in && !IsPathSeparator(*in)) {
1229 in++; /* skip chars */
1232 numsep++; /* count separators seen */
1233 in++; /* skip separator */
1237 while (*in) { /* copy to end */
1238 if (IsPathSeparator(*in)) {
1244 Dmsg4(500, "stripped=%d count=%d numsep=%d sep>count=%d\n",
1245 stripped, count, numsep, numsep>count);
1246 return stripped==count && numsep>count;
1250 * If requested strip leading components of the path so that we can
1251 * save file as if it came from a subdirectory. This is most useful
1252 * for dealing with snapshots, by removing the snapshot directory, or
1253 * in handling vendor migrations where files have been restored with
1254 * a vendor product into a subdirectory.
1256 void strip_path(FF_PKT *ff_pkt)
1258 if (!(ff_pkt->flags & FO_STRIPPATH) || ff_pkt->strip_path <= 0) {
1259 Dmsg1(200, "No strip for %s\n", ff_pkt->fname);
1262 if (!ff_pkt->fname_save) {
1263 ff_pkt->fname_save = get_pool_memory(PM_FNAME);
1264 ff_pkt->link_save = get_pool_memory(PM_FNAME);
1266 pm_strcpy(ff_pkt->fname_save, ff_pkt->fname);
1267 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1268 pm_strcpy(ff_pkt->link_save, ff_pkt->link);
1269 Dmsg2(500, "strcpy link_save=%d link=%d\n", strlen(ff_pkt->link_save),
1270 strlen(ff_pkt->link));
1271 sm_check(__FILE__, __LINE__, true);
1275 * Strip path. If it doesn't succeed put it back. If
1276 * it does, and there is a different link string,
1277 * attempt to strip the link. If it fails, back them
1279 * Do not strip symlinks.
1280 * I.e. if either stripping fails don't strip anything.
1282 if (!do_strip(ff_pkt->strip_path, ff_pkt->fname)) {
1283 unstrip_path(ff_pkt);
1286 /** Strip links but not symlinks */
1287 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1288 if (!do_strip(ff_pkt->strip_path, ff_pkt->link)) {
1289 unstrip_path(ff_pkt);
1294 Dmsg3(100, "fname=%s stripped=%s link=%s\n", ff_pkt->fname_save, ff_pkt->fname,
1298 void unstrip_path(FF_PKT *ff_pkt)
1300 if (!(ff_pkt->flags & FO_STRIPPATH) || ff_pkt->strip_path <= 0) {
1303 strcpy(ff_pkt->fname, ff_pkt->fname_save);
1304 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1305 Dmsg2(500, "strcpy link=%s link_save=%s\n", ff_pkt->link,
1307 strcpy(ff_pkt->link, ff_pkt->link_save);
1308 Dmsg2(500, "strcpy link=%d link_save=%d\n", strlen(ff_pkt->link),
1309 strlen(ff_pkt->link_save));
1310 sm_check(__FILE__, __LINE__, true);