2 Bacula® - The Network Backup Solution
4 Copyright (C) 2000-2010 Free Software Foundation Europe e.V.
6 The main author of Bacula is Kern Sibbald, with contributions from
7 many others, a complete list can be found in the file AUTHORS.
8 This program is Free Software; you can redistribute it and/or
9 modify it under the terms of version two of the GNU General Public
10 License as published by the Free Software Foundation and included
13 This program is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
23 Bacula® is a registered trademark of Kern Sibbald.
24 The licensor of Bacula is the Free Software Foundation Europe
25 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
26 Switzerland, email:ftf@fsfeurope.org.
29 * Bacula File Daemon backup.c send file attributes and data
30 * to the Storage daemon.
32 * Kern Sibbald, March MM
40 const bool have_darwin_os = true;
42 const bool have_darwin_os = false;
46 const bool have_acl = true;
48 const bool have_acl = false;
51 #if defined(HAVE_XATTR)
52 const bool have_xattr = true;
54 const bool have_xattr = false;
57 /* Forward referenced functions */
58 int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level);
59 static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, DIGEST *signature_digest);
60 bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream);
61 static bool crypto_session_start(JCR *jcr);
62 static void crypto_session_end(JCR *jcr);
63 static bool crypto_session_send(JCR *jcr, BSOCK *sd);
66 * Find all the requested files and send them
67 * to the Storage daemon.
69 * Note, we normally carry on a one-way
70 * conversation from this point on with the SD, simply blasting
71 * data to him. To properly know what is going on, we
72 * also run a "heartbeat" monitor which reads the socket and
73 * reacts accordingly (at the moment it has nothing to do
74 * except echo the heartbeat to the Director).
77 bool blast_data_to_storage_daemon(JCR *jcr, char *addr)
81 // TODO landonf: Allow user to specify encryption algorithm
83 sd = jcr->store_bsock;
85 set_jcr_job_status(jcr, JS_Running);
87 Dmsg1(300, "bfiled: opened data connection %d to stored\n", sd->m_fd);
90 CLIENT *client = (CLIENT *)GetNextRes(R_CLIENT, NULL);
94 buf_size = client->max_network_buffer_size;
96 buf_size = 0; /* use default */
98 if (!sd->set_buffer_size(buf_size, BNET_SETBUF_WRITE)) {
99 set_jcr_job_status(jcr, JS_ErrorTerminated);
100 Jmsg(jcr, M_FATAL, 0, _("Cannot set buffer size FD->SD.\n"));
104 jcr->buf_size = sd->msglen;
106 * Adjust for compression so that output buffer is
107 * 12 bytes + 0.1% larger than input buffer plus 18 bytes.
108 * This gives a bit extra plus room for the sparse addr if any.
109 * Note, we adjust the read size to be smaller so that the
110 * same output buffer can be used without growing it.
112 * The zlib compression workset is initialized here to minimize
113 * the "per file" load. The jcr member is only set, if the init
116 jcr->compress_buf_size = jcr->buf_size + ((jcr->buf_size+999) / 1000) + 30;
117 jcr->compress_buf = get_memory(jcr->compress_buf_size);
120 z_stream *pZlibStream = (z_stream*)malloc(sizeof(z_stream));
122 pZlibStream->zalloc = Z_NULL;
123 pZlibStream->zfree = Z_NULL;
124 pZlibStream->opaque = Z_NULL;
125 pZlibStream->state = Z_NULL;
127 if (deflateInit(pZlibStream, Z_DEFAULT_COMPRESSION) == Z_OK) {
128 jcr->pZLIB_compress_workset = pZlibStream;
135 if (!crypto_session_start(jcr)) {
139 set_find_options((FF_PKT *)jcr->ff, jcr->incremental, jcr->mtime);
141 /** in accurate mode, we overwrite the find_one check function */
143 set_find_changed_function((FF_PKT *)jcr->ff, accurate_check_file);
146 start_heartbeat_monitor(jcr);
149 jcr->acl_data = (acl_data_t *)malloc(sizeof(acl_data_t));
150 memset((caddr_t)jcr->acl_data, 0, sizeof(acl_data_t));
151 jcr->acl_data->content = get_pool_memory(PM_MESSAGE);
155 jcr->xattr_data = (xattr_data_t *)malloc(sizeof(xattr_data_t));
156 memset((caddr_t)jcr->xattr_data, 0, sizeof(xattr_data_t));
157 jcr->xattr_data->content = get_pool_memory(PM_MESSAGE);
160 /** Subroutine save_file() is called for each file */
161 if (!find_files(jcr, (FF_PKT *)jcr->ff, save_file, plugin_save)) {
162 ok = false; /* error */
163 set_jcr_job_status(jcr, JS_ErrorTerminated);
166 if (have_acl && jcr->acl_data->nr_errors > 0) {
167 Jmsg(jcr, M_ERROR, 0, _("Encountered %ld acl errors while doing backup\n"),
168 jcr->acl_data->nr_errors);
170 if (have_xattr && jcr->xattr_data->nr_errors > 0) {
171 Jmsg(jcr, M_ERROR, 0, _("Encountered %ld xattr errors while doing backup\n"),
172 jcr->xattr_data->nr_errors);
175 accurate_finish(jcr); /* send deleted or base file list to SD */
177 stop_heartbeat_monitor(jcr);
179 sd->signal(BNET_EOD); /* end of sending data */
181 if (have_acl && jcr->acl_data) {
182 free_pool_memory(jcr->acl_data->content);
184 jcr->acl_data = NULL;
186 if (have_xattr && jcr->xattr_data) {
187 free_pool_memory(jcr->xattr_data->content);
188 free(jcr->xattr_data);
189 jcr->xattr_data = NULL;
195 if (jcr->compress_buf) {
196 free_pool_memory(jcr->compress_buf);
197 jcr->compress_buf = NULL;
199 if (jcr->pZLIB_compress_workset) {
200 /* Free the zlib stream */
202 deflateEnd((z_stream *)jcr->pZLIB_compress_workset);
204 free (jcr->pZLIB_compress_workset);
205 jcr->pZLIB_compress_workset = NULL;
207 crypto_session_end(jcr);
210 Dmsg1(100, "end blast_data ok=%d\n", ok);
214 static bool crypto_session_start(JCR *jcr)
216 crypto_cipher_t cipher = CRYPTO_CIPHER_AES_128_CBC;
219 * Create encryption session data and a cached, DER-encoded session data
220 * structure. We use a single session key for each backup, so we'll encode
221 * the session data only once.
223 if (jcr->crypto.pki_encrypt) {
226 /** Create per-job session encryption context */
227 jcr->crypto.pki_session = crypto_session_new(cipher, jcr->crypto.pki_recipients);
229 /** Get the session data size */
230 if (!crypto_session_encode(jcr->crypto.pki_session, (uint8_t *)0, &size)) {
231 Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n"));
235 /** Allocate buffer */
236 jcr->crypto.pki_session_encoded = get_memory(size);
238 /** Encode session data */
239 if (!crypto_session_encode(jcr->crypto.pki_session, (uint8_t *)jcr->crypto.pki_session_encoded, &size)) {
240 Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n"));
244 /** ... and store the encoded size */
245 jcr->crypto.pki_session_encoded_size = size;
247 /** Allocate the encryption/decryption buffer */
248 jcr->crypto.crypto_buf = get_memory(CRYPTO_CIPHER_MAX_BLOCK_SIZE);
253 static void crypto_session_end(JCR *jcr)
255 if (jcr->crypto.crypto_buf) {
256 free_pool_memory(jcr->crypto.crypto_buf);
257 jcr->crypto.crypto_buf = NULL;
259 if (jcr->crypto.pki_session) {
260 crypto_session_free(jcr->crypto.pki_session);
262 if (jcr->crypto.pki_session_encoded) {
263 free_pool_memory(jcr->crypto.pki_session_encoded);
264 jcr->crypto.pki_session_encoded = NULL;
268 static bool crypto_session_send(JCR *jcr, BSOCK *sd)
272 /** Send our header */
273 Dmsg2(100, "Send hdr fi=%ld stream=%d\n", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA);
274 sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA);
277 sd->msg = jcr->crypto.pki_session_encoded;
278 sd->msglen = jcr->crypto.pki_session_encoded_size;
279 jcr->JobBytes += sd->msglen;
281 Dmsg1(100, "Send data len=%d\n", sd->msglen);
284 sd->signal(BNET_EOD);
290 * Called here by find() for each file included.
291 * This is a callback. The original is find_files() above.
293 * Send the file and its data to the Storage daemon.
297 * -1 to ignore file/directory (not used here)
299 int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level)
301 bool do_read = false;
302 bool plugin_started = false;
303 int stat, data_stream;
305 DIGEST *digest = NULL;
306 DIGEST *signing_digest = NULL;
307 int digest_stream = STREAM_NONE;
308 SIGNATURE *sig = NULL;
309 bool has_file_data = false;
310 // TODO landonf: Allow the user to specify the digest algorithm
312 crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA256;
314 crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA1;
316 BSOCK *sd = jcr->store_bsock;
318 if (jcr->is_job_canceled()) {
322 jcr->num_files_examined++; /* bump total file count */
324 switch (ff_pkt->type) {
325 case FT_LNKSAVED: /* Hard linked, file already saved */
326 Dmsg2(130, "FT_LNKSAVED hard link: %s => %s\n", ff_pkt->fname, ff_pkt->link);
329 Dmsg1(130, "FT_REGE saving: %s\n", ff_pkt->fname);
330 has_file_data = true;
333 Dmsg1(130, "FT_REG saving: %s\n", ff_pkt->fname);
334 has_file_data = true;
337 Dmsg2(130, "FT_LNK saving: %s -> %s\n", ff_pkt->fname, ff_pkt->link);
339 case FT_RESTORE_FIRST:
340 Dmsg1(100, "FT_RESTORE_FIRST saving: %s\n", ff_pkt->fname);
343 jcr->num_files_examined--; /* correct file count */
344 return 1; /* not used */
346 Jmsg(jcr, M_INFO, 1, _(" Recursion turned off. Will not descend from %s into %s\n"),
347 ff_pkt->top_fname, ff_pkt->fname);
348 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
351 /* Suppress message for /dev filesystems */
352 if (!is_in_fileset(ff_pkt)) {
353 Jmsg(jcr, M_INFO, 1, _(" %s is a different filesystem. Will not descend from %s into %s\n"),
354 ff_pkt->fname, ff_pkt->top_fname, ff_pkt->fname);
356 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
359 Jmsg(jcr, M_INFO, 1, _(" Disallowed filesystem. Will not descend from %s into %s\n"),
360 ff_pkt->top_fname, ff_pkt->fname);
361 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
364 Jmsg(jcr, M_INFO, 1, _(" Disallowed drive type. Will not descend into %s\n"),
369 Dmsg1(130, "FT_DIREND: %s\n", ff_pkt->link);
372 Dmsg1(130, "FT_SPEC saving: %s\n", ff_pkt->fname);
373 if (S_ISSOCK(ff_pkt->statp.st_mode)) {
374 Jmsg(jcr, M_SKIPPED, 1, _(" Socket file skipped: %s\n"), ff_pkt->fname);
379 Dmsg1(130, "FT_RAW saving: %s\n", ff_pkt->fname);
380 has_file_data = true;
383 Dmsg1(130, "FT_FIFO saving: %s\n", ff_pkt->fname);
387 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not access \"%s\": ERR=%s\n"), ff_pkt->fname,
388 be.bstrerror(ff_pkt->ff_errno));
394 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not follow link \"%s\": ERR=%s\n"),
395 ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
401 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not stat \"%s\": ERR=%s\n"), ff_pkt->fname,
402 be.bstrerror(ff_pkt->ff_errno));
408 Jmsg(jcr, M_SKIPPED, 1, _(" Unchanged file skipped: %s\n"), ff_pkt->fname);
411 Jmsg(jcr, M_NOTSAVED, 0, _(" Archive file not saved: %s\n"), ff_pkt->fname);
415 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not open directory \"%s\": ERR=%s\n"),
416 ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
421 Jmsg(jcr, M_NOTSAVED, 0, _(" Unknown file type %d; not saved: %s\n"),
422 ff_pkt->type, ff_pkt->fname);
427 Dmsg1(130, "bfiled: sending %s to stored\n", ff_pkt->fname);
429 /** Digests and encryption are only useful if there's file data */
432 * Setup for digest handling. If this fails, the digest will be set to NULL
433 * and not used. Note, the digest (file hash) can be any one of the four
436 * The signing digest is a single algorithm depending on
437 * whether or not we have SHA2.
438 * ****FIXME**** the signing algoritm should really be
439 * determined a different way!!!!!! What happens if
440 * sha2 was available during backup but not restore?
442 if (ff_pkt->flags & FO_MD5) {
443 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_MD5);
444 digest_stream = STREAM_MD5_DIGEST;
446 } else if (ff_pkt->flags & FO_SHA1) {
447 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA1);
448 digest_stream = STREAM_SHA1_DIGEST;
450 } else if (ff_pkt->flags & FO_SHA256) {
451 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA256);
452 digest_stream = STREAM_SHA256_DIGEST;
454 } else if (ff_pkt->flags & FO_SHA512) {
455 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA512);
456 digest_stream = STREAM_SHA512_DIGEST;
459 /** Did digest initialization fail? */
460 if (digest_stream != STREAM_NONE && digest == NULL) {
461 Jmsg(jcr, M_WARNING, 0, _("%s digest initialization failed\n"),
462 stream_to_ascii(digest_stream));
466 * Set up signature digest handling. If this fails, the signature digest
467 * will be set to NULL and not used.
469 /* TODO landonf: We should really only calculate the digest once, for
470 * both verification and signing.
472 if (jcr->crypto.pki_sign) {
473 signing_digest = crypto_digest_new(jcr, signing_algorithm);
475 /** Full-stop if a failure occurred initializing the signature digest */
476 if (signing_digest == NULL) {
477 Jmsg(jcr, M_NOTSAVED, 0, _("%s signature digest initialization failed\n"),
478 stream_to_ascii(signing_algorithm));
484 /** Enable encryption */
485 if (jcr->crypto.pki_encrypt) {
486 ff_pkt->flags |= FO_ENCRYPT;
490 /** Initialize the file descriptor we use for data and other streams. */
492 if (ff_pkt->flags & FO_PORTABLE) {
493 set_portable_backup(&ff_pkt->bfd); /* disable Win32 BackupRead() */
495 if (ff_pkt->cmd_plugin) {
496 /* Tell bfile that it needs to call plugin */
497 if (!set_cmd_plugin(&ff_pkt->bfd, jcr)) {
500 send_plugin_name(jcr, sd, true); /* signal start of plugin data */
501 plugin_started = true;
504 /** Send attributes -- must be done after binit() */
505 if (!encode_and_send_attributes(jcr, ff_pkt, data_stream)) {
508 /** Meta data only for restore object */
509 if (ff_pkt->type == FT_RESTORE_FIRST) {
513 /** Set up the encryption context and send the session data to the SD */
514 if (has_file_data && jcr->crypto.pki_encrypt) {
515 if (!crypto_session_send(jcr, sd)) {
521 * Open any file with data that we intend to save, then save it.
523 * Note, if is_win32_backup, we must open the Directory so that
524 * the BackupRead will save its permissions and ownership streams.
526 if (ff_pkt->type != FT_LNKSAVED && S_ISREG(ff_pkt->statp.st_mode)) {
528 do_read = !is_portable_backup(&ff_pkt->bfd) || ff_pkt->statp.st_size > 0;
530 do_read = ff_pkt->statp.st_size > 0;
532 } else if (ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO ||
533 ff_pkt->type == FT_REPARSE ||
534 (!is_portable_backup(&ff_pkt->bfd) && ff_pkt->type == FT_DIREND)) {
538 if (ff_pkt->cmd_plugin) {
542 Dmsg2(100, "type=%d do_read=%d\n", ff_pkt->type, do_read);
546 if (ff_pkt->type == FT_FIFO) {
547 tid = start_thread_timer(jcr, pthread_self(), 60);
551 int noatime = ff_pkt->flags & FO_NOATIME ? O_NOATIME : 0;
552 ff_pkt->bfd.reparse_point = ff_pkt->type == FT_REPARSE;
553 if (bopen(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY | noatime, 0) < 0) {
554 ff_pkt->ff_errno = errno;
556 Jmsg(jcr, M_NOTSAVED, 0, _(" Cannot open \"%s\": ERR=%s.\n"), ff_pkt->fname,
560 stop_thread_timer(tid);
566 stop_thread_timer(tid);
570 stat = send_data(jcr, data_stream, ff_pkt, digest, signing_digest);
572 if (ff_pkt->flags & FO_CHKCHANGES) {
573 has_file_changed(jcr, ff_pkt);
576 bclose(&ff_pkt->bfd);
583 if (have_darwin_os) {
584 /** Regular files can have resource forks and Finder Info */
585 if (ff_pkt->type != FT_LNKSAVED && (S_ISREG(ff_pkt->statp.st_mode) &&
586 ff_pkt->flags & FO_HFSPLUS)) {
587 if (ff_pkt->hfsinfo.rsrclength > 0) {
590 if (!bopen_rsrc(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0) < 0) {
591 ff_pkt->ff_errno = errno;
593 Jmsg(jcr, M_NOTSAVED, -1, _(" Cannot open resource fork for \"%s\": ERR=%s.\n"),
594 ff_pkt->fname, be.bstrerror());
596 if (is_bopen(&ff_pkt->bfd)) {
597 bclose(&ff_pkt->bfd);
601 flags = ff_pkt->flags;
602 ff_pkt->flags &= ~(FO_GZIP|FO_SPARSE);
603 if (flags & FO_ENCRYPT) {
604 rsrc_stream = STREAM_ENCRYPTED_MACOS_FORK_DATA;
606 rsrc_stream = STREAM_MACOS_FORK_DATA;
608 stat = send_data(jcr, rsrc_stream, ff_pkt, digest, signing_digest);
609 ff_pkt->flags = flags;
610 bclose(&ff_pkt->bfd);
616 Dmsg1(300, "Saving Finder Info for \"%s\"\n", ff_pkt->fname);
617 sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_HFSPLUS_ATTRIBUTES);
618 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
619 pm_memcpy(sd->msg, ff_pkt->hfsinfo.fndrinfo, 32);
622 crypto_digest_update(digest, (uint8_t *)sd->msg, sd->msglen);
624 if (signing_digest) {
625 crypto_digest_update(signing_digest, (uint8_t *)sd->msg, sd->msglen);
628 sd->signal(BNET_EOD);
633 * Save ACLs when requested and available for anything not being a symlink and not being a plugin.
636 if (ff_pkt->flags & FO_ACL && ff_pkt->type != FT_LNK && !ff_pkt->cmd_plugin) {
637 switch (build_acl_streams(jcr, ff_pkt)) {
638 case bacl_exit_fatal:
640 case bacl_exit_error:
642 * Non-fatal errors, count them and when the number is under ACL_REPORT_ERR_MAX_PER_JOB
643 * print the error message set by the lower level routine in jcr->errmsg.
645 if (jcr->acl_data->nr_errors < ACL_REPORT_ERR_MAX_PER_JOB) {
646 Jmsg(jcr, M_ERROR, 0, "%s", jcr->errmsg);
648 jcr->acl_data->nr_errors++;
657 * Save Extended Attributes when requested and available for all files not being a plugin.
660 if (ff_pkt->flags & FO_XATTR && !ff_pkt->cmd_plugin) {
661 switch (build_xattr_streams(jcr, ff_pkt)) {
662 case bxattr_exit_fatal:
664 case bxattr_exit_error:
666 * Non-fatal errors, count them and when the number is under XATTR_REPORT_ERR_MAX_PER_JOB
667 * print the error message set by the lower level routine in jcr->errmsg.
669 if (jcr->xattr_data->nr_errors < XATTR_REPORT_ERR_MAX_PER_JOB) {
670 Jmsg(jcr, M_ERROR, 0, "%s", jcr->errmsg);
672 jcr->xattr_data->nr_errors++;
680 /** Terminate the signing digest and send it to the Storage daemon */
681 if (signing_digest) {
684 if ((sig = crypto_sign_new(jcr)) == NULL) {
685 Jmsg(jcr, M_FATAL, 0, _("Failed to allocate memory for crypto signature.\n"));
689 if (!crypto_sign_add_signer(sig, signing_digest, jcr->crypto.pki_keypair)) {
690 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
694 /** Get signature size */
695 if (!crypto_sign_encode(sig, NULL, &size)) {
696 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
700 /** Grow the bsock buffer to fit our message if necessary */
701 if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
702 sd->msg = realloc_pool_memory(sd->msg, size);
705 /** Send our header */
706 sd->fsend("%ld %ld 0", jcr->JobFiles, STREAM_SIGNED_DIGEST);
707 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
709 /** Encode signature data */
710 if (!crypto_sign_encode(sig, (uint8_t *)sd->msg, &size)) {
711 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
717 sd->signal(BNET_EOD); /* end of checksum */
720 /** Terminate any digest and send it to Storage daemon */
724 sd->fsend("%ld %d 0", jcr->JobFiles, digest_stream);
725 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
727 size = CRYPTO_DIGEST_MAX_SIZE;
729 /** Grow the bsock buffer to fit our message if necessary */
730 if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
731 sd->msg = realloc_pool_memory(sd->msg, size);
734 if (!crypto_digest_finalize(digest, (uint8_t *)sd->msg, &size)) {
735 Jmsg(jcr, M_FATAL, 0, _("An error occurred finalizing signing the stream.\n"));
741 sd->signal(BNET_EOD); /* end of checksum */
745 rtnstat = 1; /* good return */
748 if (ff_pkt->cmd_plugin && plugin_started) {
749 send_plugin_name(jcr, sd, false); /* signal end of plugin data */
752 crypto_digest_free(digest);
754 if (signing_digest) {
755 crypto_digest_free(signing_digest);
758 crypto_sign_free(sig);
764 * Send data read from an already open file descriptor.
766 * We return 1 on sucess and 0 on errors.
769 * We use ff_pkt->statp.st_size when FO_SPARSE to know when to stop
771 * Currently this is not a problem as the only other stream, resource forks,
772 * are not handled as sparse files.
774 static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest,
775 DIGEST *signing_digest)
777 BSOCK *sd = jcr->store_bsock;
778 uint64_t fileAddr = 0; /* file address */
780 int32_t rsize = jcr->buf_size; /* read buffer size */
782 CIPHER_CONTEXT *cipher_ctx = NULL; /* Quell bogus uninitialized warnings */
783 const uint8_t *cipher_input;
784 uint32_t cipher_input_len;
785 uint32_t cipher_block_size;
786 uint32_t encrypted_len;
787 #ifdef FD_NO_SEND_TEST
792 rbuf = sd->msg; /* read buffer */
793 wbuf = sd->msg; /* write buffer */
794 cipher_input = (uint8_t *)rbuf; /* encrypt uncompressed data */
796 Dmsg1(300, "Saving data, type=%d\n", ff_pkt->type);
799 uLong compress_len = 0;
800 uLong max_compress_len = 0;
801 const Bytef *cbuf = NULL;
804 if (ff_pkt->flags & FO_GZIP) {
805 if (ff_pkt->flags & FO_SPARSE) {
806 cbuf = (Bytef *)jcr->compress_buf + SPARSE_FADDR_SIZE;
807 max_compress_len = jcr->compress_buf_size - SPARSE_FADDR_SIZE;
809 cbuf = (Bytef *)jcr->compress_buf;
810 max_compress_len = jcr->compress_buf_size; /* set max length */
812 wbuf = jcr->compress_buf; /* compressed output here */
813 cipher_input = (uint8_t *)jcr->compress_buf; /* encrypt compressed data */
816 * Only change zlib parameters if there is no pending operation.
817 * This should never happen as deflatereset is called after each
821 if (((z_stream*)jcr->pZLIB_compress_workset)->total_in == 0) {
822 /** set gzip compression level - must be done per file */
823 if ((zstat=deflateParams((z_stream*)jcr->pZLIB_compress_workset,
824 ff_pkt->GZIP_level, Z_DEFAULT_STRATEGY)) != Z_OK) {
825 Jmsg(jcr, M_FATAL, 0, _("Compression deflateParams error: %d\n"), zstat);
826 set_jcr_job_status(jcr, JS_ErrorTerminated);
832 const uint32_t max_compress_len = 0;
835 if (ff_pkt->flags & FO_ENCRYPT) {
836 if (ff_pkt->flags & FO_SPARSE) {
837 Jmsg0(jcr, M_FATAL, 0, _("Encrypting sparse data not supported.\n"));
840 /** Allocate the cipher context */
841 if ((cipher_ctx = crypto_cipher_new(jcr->crypto.pki_session, true,
842 &cipher_block_size)) == NULL) {
843 /* Shouldn't happen! */
844 Jmsg0(jcr, M_FATAL, 0, _("Failed to initialize encryption context.\n"));
849 * Grow the crypto buffer, if necessary.
850 * crypto_cipher_update() will buffer up to (cipher_block_size - 1).
851 * We grow crypto_buf to the maximum number of blocks that
852 * could be returned for the given read buffer size.
853 * (Using the larger of either rsize or max_compress_len)
855 jcr->crypto.crypto_buf = check_pool_memory_size(jcr->crypto.crypto_buf,
856 (MAX(rsize + (int)sizeof(uint32_t), (int32_t)max_compress_len) +
857 cipher_block_size - 1) / cipher_block_size * cipher_block_size);
859 wbuf = jcr->crypto.crypto_buf; /* Encrypted, possibly compressed output here. */
863 * Send Data header to Storage daemon
864 * <file-index> <stream> <info>
866 if (!sd->fsend("%ld %d 0", jcr->JobFiles, stream)) {
867 if (!jcr->is_job_canceled()) {
868 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
873 Dmsg1(300, ">stored: datahdr %s\n", sd->msg);
876 * Make space at beginning of buffer for fileAddr because this
877 * same buffer will be used for writing if compression is off.
879 if (ff_pkt->flags & FO_SPARSE) {
880 rbuf += SPARSE_FADDR_SIZE;
881 rsize -= SPARSE_FADDR_SIZE;
882 #ifdef HAVE_FREEBSD_OS
884 * To read FreeBSD partitions, the read size must be
887 rsize = (rsize/512) * 512;
891 /** a RAW device read on win32 only works if the buffer is a multiple of 512 */
893 if (S_ISBLK(ff_pkt->statp.st_mode))
894 rsize = (rsize/512) * 512;
900 while ((sd->msglen=(uint32_t)bread(&ff_pkt->bfd, rbuf, rsize)) > 0) {
902 /** Check for sparse blocks */
903 if (ff_pkt->flags & FO_SPARSE) {
905 bool allZeros = false;
906 if ((sd->msglen == rsize &&
907 fileAddr+sd->msglen < (uint64_t)ff_pkt->statp.st_size) ||
908 ((ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO) &&
909 (uint64_t)ff_pkt->statp.st_size == 0)) {
910 allZeros = is_buf_zero(rbuf, rsize);
913 /** Put file address as first data in buffer */
914 ser_begin(wbuf, SPARSE_FADDR_SIZE);
915 ser_uint64(fileAddr); /* store fileAddr in begin of buffer */
917 fileAddr += sd->msglen; /* update file address */
918 /** Skip block of all zeros */
920 continue; /* skip block of zeros */
924 jcr->ReadBytes += sd->msglen; /* count bytes read */
926 /** Uncompressed cipher input length */
927 cipher_input_len = sd->msglen;
929 /** Update checksum if requested */
931 crypto_digest_update(digest, (uint8_t *)rbuf, sd->msglen);
934 /** Update signing digest if requested */
935 if (signing_digest) {
936 crypto_digest_update(signing_digest, (uint8_t *)rbuf, sd->msglen);
940 /** Do compression if turned on */
941 if (ff_pkt->flags & FO_GZIP && jcr->pZLIB_compress_workset) {
942 Dmsg3(400, "cbuf=0x%x rbuf=0x%x len=%u\n", cbuf, rbuf, sd->msglen);
944 ((z_stream*)jcr->pZLIB_compress_workset)->next_in = (Bytef *)rbuf;
945 ((z_stream*)jcr->pZLIB_compress_workset)->avail_in = sd->msglen;
946 ((z_stream*)jcr->pZLIB_compress_workset)->next_out = (Bytef *)cbuf;
947 ((z_stream*)jcr->pZLIB_compress_workset)->avail_out = max_compress_len;
949 if ((zstat=deflate((z_stream*)jcr->pZLIB_compress_workset, Z_FINISH)) != Z_STREAM_END) {
950 Jmsg(jcr, M_FATAL, 0, _("Compression deflate error: %d\n"), zstat);
951 set_jcr_job_status(jcr, JS_ErrorTerminated);
954 compress_len = ((z_stream*)jcr->pZLIB_compress_workset)->total_out;
955 /** reset zlib stream to be able to begin from scratch again */
956 if ((zstat=deflateReset((z_stream*)jcr->pZLIB_compress_workset)) != Z_OK) {
957 Jmsg(jcr, M_FATAL, 0, _("Compression deflateReset error: %d\n"), zstat);
958 set_jcr_job_status(jcr, JS_ErrorTerminated);
962 Dmsg2(400, "compressed len=%d uncompressed len=%d\n", compress_len,
965 sd->msglen = compress_len; /* set compressed length */
966 cipher_input_len = compress_len;
970 * Note, here we prepend the current record length to the beginning
971 * of the encrypted data. This is because both sparse and compression
972 * restore handling want records returned to them with exactly the
973 * same number of bytes that were processed in the backup handling.
974 * That is, both are block filters rather than a stream. When doing
975 * compression, the compression routines may buffer data, so that for
976 * any one record compressed, when it is decompressed the same size
977 * will not be obtained. Of course, the buffered data eventually comes
978 * out in subsequent crypto_cipher_update() calls or at least
979 * when crypto_cipher_finalize() is called. Unfortunately, this
980 * "feature" of encryption enormously complicates the restore code.
982 if (ff_pkt->flags & FO_ENCRYPT) {
983 uint32_t initial_len = 0;
986 if (ff_pkt->flags & FO_SPARSE) {
987 cipher_input_len += SPARSE_FADDR_SIZE;
990 /** Encrypt the length of the input block */
991 uint8_t packet_len[sizeof(uint32_t)];
993 ser_begin(packet_len, sizeof(uint32_t));
994 ser_uint32(cipher_input_len); /* store data len in begin of buffer */
995 Dmsg1(20, "Encrypt len=%d\n", cipher_input_len);
997 if (!crypto_cipher_update(cipher_ctx, packet_len, sizeof(packet_len),
998 (uint8_t *)jcr->crypto.crypto_buf, &initial_len)) {
999 /** Encryption failed. Shouldn't happen. */
1000 Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
1004 /** Encrypt the input block */
1005 if (crypto_cipher_update(cipher_ctx, cipher_input, cipher_input_len,
1006 (uint8_t *)&jcr->crypto.crypto_buf[initial_len], &encrypted_len)) {
1007 if ((initial_len + encrypted_len) == 0) {
1008 /** No full block of data available, read more data */
1011 Dmsg2(400, "encrypted len=%d unencrypted len=%d\n", encrypted_len,
1013 sd->msglen = initial_len + encrypted_len; /* set encrypted length */
1015 /** Encryption failed. Shouldn't happen. */
1016 Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
1021 /* Send the buffer to the Storage daemon */
1022 if (ff_pkt->flags & FO_SPARSE) {
1023 sd->msglen += SPARSE_FADDR_SIZE; /* include fileAddr in size */
1025 sd->msg = wbuf; /* set correct write buffer */
1027 if (!jcr->is_job_canceled()) {
1028 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1033 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
1035 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
1036 sd->msg = msgsave; /* restore read buffer */
1038 } /* end while read file data */
1040 if (sd->msglen < 0) { /* error */
1042 Jmsg(jcr, M_ERROR, 0, _("Read error on file %s. ERR=%s\n"),
1043 ff_pkt->fname, be.bstrerror(ff_pkt->bfd.berrno));
1044 if (jcr->JobErrors++ > 1000) { /* insanity check */
1045 Jmsg(jcr, M_FATAL, 0, _("Too many errors. JobErrors=%d.\n"), jcr->JobErrors);
1047 } else if (ff_pkt->flags & FO_ENCRYPT) {
1049 * For encryption, we must call finalize to push out any
1052 if (!crypto_cipher_finalize(cipher_ctx, (uint8_t *)jcr->crypto.crypto_buf,
1054 /* Padding failed. Shouldn't happen. */
1055 Jmsg(jcr, M_FATAL, 0, _("Encryption padding error\n"));
1059 /** Note, on SSL pre-0.9.7, there is always some output */
1060 if (encrypted_len > 0) {
1061 sd->msglen = encrypted_len; /* set encrypted length */
1062 sd->msg = jcr->crypto.crypto_buf; /* set correct write buffer */
1064 if (!jcr->is_job_canceled()) {
1065 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1070 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
1071 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
1072 sd->msg = msgsave; /* restore bnet buffer */
1076 if (!sd->signal(BNET_EOD)) { /* indicate end of file data */
1077 if (!jcr->is_job_canceled()) {
1078 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1084 /** Free the cipher context */
1086 crypto_cipher_free(cipher_ctx);
1091 /** Free the cipher context */
1093 crypto_cipher_free(cipher_ctx);
1096 sd->msg = msgsave; /* restore bnet buffer */
1101 bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream)
1103 BSOCK *sd = jcr->store_bsock;
1104 char attribs[MAXSTRING];
1105 char attribsExBuf[MAXSTRING];
1109 #ifdef FD_NO_SEND_TEST
1113 Dmsg1(300, "encode_and_send_attrs fname=%s\n", ff_pkt->fname);
1114 /** Find what data stream we will use, then encode the attributes */
1115 if ((data_stream = select_data_stream(ff_pkt)) == STREAM_NONE) {
1116 /* This should not happen */
1117 Jmsg0(jcr, M_FATAL, 0, _("Invalid file flags, no supported data stream type.\n"));
1120 encode_stat(attribs, &ff_pkt->statp, ff_pkt->LinkFI, data_stream);
1122 /** Now possibly extend the attributes */
1123 if (ff_pkt->type == FT_RESTORE_FIRST) {
1125 * For restore objects, we return the object in the extended
1128 attribsEx = ff_pkt->object;
1129 attr_stream = STREAM_UNIX_ATTRIBUTES_EX;
1131 attribsEx = attribsExBuf;
1132 attr_stream = encode_attribsEx(jcr, attribsEx, ff_pkt);
1135 Dmsg3(300, "File %s\nattribs=%s\nattribsEx=%s\n", ff_pkt->fname, attribs, attribsEx);
1138 jcr->JobFiles++; /* increment number of files sent */
1139 ff_pkt->FileIndex = jcr->JobFiles; /* return FileIndex */
1140 pm_strcpy(jcr->last_fname, ff_pkt->fname);
1144 * Send Attributes header to Storage daemon
1145 * <file-index> <stream> <info>
1147 if (!sd->fsend("%ld %d 0", jcr->JobFiles, attr_stream)) {
1148 if (!jcr->is_job_canceled()) {
1149 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1154 Dmsg1(300, ">stored: attrhdr %s\n", sd->msg);
1157 * Send file attributes to Storage daemon
1160 * Filename (full path)
1161 * Encoded attributes
1162 * Link name (if type==FT_LNK or FT_LNKSAVED)
1163 * Encoded extended-attributes (for Win32)
1165 * For a directory, link is the same as fname, but with trailing
1166 * slash. For a linked file, link is the link.
1168 if (ff_pkt->type != FT_DELETED) { /* already stripped */
1171 switch (ff_pkt->type) {
1174 Dmsg2(300, "Link %s to %s\n", ff_pkt->fname, ff_pkt->link);
1175 stat = sd->fsend("%ld %d %s%c%s%c%s%c%s%c", jcr->JobFiles,
1176 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, ff_pkt->link, 0,
1181 /* Here link is the canonical filename (i.e. with trailing slash) */
1182 stat = sd->fsend("%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
1183 ff_pkt->type, ff_pkt->link, 0, attribs, 0, 0, attribsEx, 0);
1185 case FT_RESTORE_FIRST:
1187 * Note, we edit everything as we do for the default case, but the
1188 * object is tacked on to the end in place of the extended attributes,
1189 * but we do a memcpy so that the object can be a binary object.
1191 Dmsg6(100, "Type=%d DataStream=%d attrStream=%d File=%s\nattribs=%s\nattribsEx=%s",
1192 ff_pkt->type, data_stream, STREAM_UNIX_ATTRIBUTES_EX,
1193 ff_pkt->fname, attribs, ff_pkt->object);
1194 sd->msglen = Mmsg(sd->msg, "%ld %d %s%c%s%c%c",
1195 jcr->JobFiles, ff_pkt->type, ff_pkt->fname, 0, attribs, 0, 0);
1196 sd->msg = check_pool_memory_size(sd->msg, sd->msglen + ff_pkt->object_len + 1);
1197 memcpy(sd->msg + sd->msglen, ff_pkt->object, ff_pkt->object_len);
1198 sd->msglen += ff_pkt->object_len;
1202 stat = sd->fsend("%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
1203 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, 0, attribsEx, 0);
1206 if (ff_pkt->type != FT_DELETED) {
1207 unstrip_path(ff_pkt);
1210 Dmsg2(300, ">stored: attr len=%d: %s\n", sd->msglen, sd->msg);
1211 if (!stat && !jcr->is_job_canceled()) {
1212 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1215 sd->signal(BNET_EOD); /* indicate end of attributes data */
1220 * Do in place strip of path
1222 static bool do_strip(int count, char *in)
1228 /** Copy to first path separator -- Win32 might have c: ... */
1229 while (*in && !IsPathSeparator(*in)) {
1233 numsep++; /* one separator seen */
1234 for (stripped=0; stripped<count && *in; stripped++) {
1235 while (*in && !IsPathSeparator(*in)) {
1236 in++; /* skip chars */
1239 numsep++; /* count separators seen */
1240 in++; /* skip separator */
1244 while (*in) { /* copy to end */
1245 if (IsPathSeparator(*in)) {
1251 Dmsg4(500, "stripped=%d count=%d numsep=%d sep>count=%d\n",
1252 stripped, count, numsep, numsep>count);
1253 return stripped==count && numsep>count;
1257 * If requested strip leading components of the path so that we can
1258 * save file as if it came from a subdirectory. This is most useful
1259 * for dealing with snapshots, by removing the snapshot directory, or
1260 * in handling vendor migrations where files have been restored with
1261 * a vendor product into a subdirectory.
1263 void strip_path(FF_PKT *ff_pkt)
1265 if (!(ff_pkt->flags & FO_STRIPPATH) || ff_pkt->strip_path <= 0) {
1266 Dmsg1(200, "No strip for %s\n", ff_pkt->fname);
1269 if (!ff_pkt->fname_save) {
1270 ff_pkt->fname_save = get_pool_memory(PM_FNAME);
1271 ff_pkt->link_save = get_pool_memory(PM_FNAME);
1273 pm_strcpy(ff_pkt->fname_save, ff_pkt->fname);
1274 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1275 pm_strcpy(ff_pkt->link_save, ff_pkt->link);
1276 Dmsg2(500, "strcpy link_save=%d link=%d\n", strlen(ff_pkt->link_save),
1277 strlen(ff_pkt->link));
1278 sm_check(__FILE__, __LINE__, true);
1282 * Strip path. If it doesn't succeed put it back. If
1283 * it does, and there is a different link string,
1284 * attempt to strip the link. If it fails, back them
1286 * Do not strip symlinks.
1287 * I.e. if either stripping fails don't strip anything.
1289 if (!do_strip(ff_pkt->strip_path, ff_pkt->fname)) {
1290 unstrip_path(ff_pkt);
1293 /** Strip links but not symlinks */
1294 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1295 if (!do_strip(ff_pkt->strip_path, ff_pkt->link)) {
1296 unstrip_path(ff_pkt);
1301 Dmsg3(100, "fname=%s stripped=%s link=%s\n", ff_pkt->fname_save, ff_pkt->fname,
1305 void unstrip_path(FF_PKT *ff_pkt)
1307 if (!(ff_pkt->flags & FO_STRIPPATH) || ff_pkt->strip_path <= 0) {
1310 strcpy(ff_pkt->fname, ff_pkt->fname_save);
1311 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1312 Dmsg2(500, "strcpy link=%s link_save=%s\n", ff_pkt->link,
1314 strcpy(ff_pkt->link, ff_pkt->link_save);
1315 Dmsg2(500, "strcpy link=%d link_save=%d\n", strlen(ff_pkt->link),
1316 strlen(ff_pkt->link_save));
1317 sm_check(__FILE__, __LINE__, true);