2 Bacula® - The Network Backup Solution
4 Copyright (C) 2000-2008 Free Software Foundation Europe e.V.
6 The main author of Bacula is Kern Sibbald, with contributions from
7 many others, a complete list can be found in the file AUTHORS.
8 This program is Free Software; you can redistribute it and/or
9 modify it under the terms of version two of the GNU General Public
10 License as published by the Free Software Foundation and included
13 This program is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
23 Bacula® is a registered trademark of John Walker.
24 The licensor of Bacula is the Free Software Foundation Europe
25 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
26 Switzerland, email:ftf@fsfeurope.org.
29 * Bacula File Daemon backup.c send file attributes and data
30 * to the Storage daemon.
32 * Kern Sibbald, March MM
40 #include "lib/htable.h"
42 /* Forward referenced functions */
43 int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level);
44 static void strip_path(FF_PKT *ff_pkt);
45 static void unstrip_path(FF_PKT *ff_pkt);
46 static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, DIGEST *signature_digest);
47 static bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream);
48 static bool read_and_send_acl(JCR *jcr, int acltype, int stream);
49 static bool crypto_session_start(JCR *jcr);
50 static void crypto_session_end(JCR *jcr);
51 static bool crypto_session_send(JCR *jcr, BSOCK *sd);
53 typedef struct CurFile {
61 #define accurate_mark_file_as_seen(elt) ((elt)->seen = 1)
62 #define accurate_file_has_been_seen(elt) ((elt)->seen)
65 * This function is called for each file seen in fileset.
66 * We check in file_list hash if fname have been backuped
67 * the last time. After we can compare Lstat field.
68 * Full Lstat usage have been removed on 6612
70 bool accurate_check_file(JCR *jcr, FF_PKT *ff_pkt)
76 if (!jcr->accurate || jcr->JobLevel == L_FULL) {
82 if (S_ISDIR(ff_pkt->statp.st_mode)) {
85 fname = ff_pkt->fname;
88 elt = (CurFile *)jcr->file_list->lookup(fname);
91 Dmsg1(500, "accurate %s = yes (not found)\n", fname);
96 if (accurate_file_has_been_seen(elt)) {
97 Dmsg1(500, "accurate %s = no (already seen)\n", fname);
101 if (elt->mtime != ff_pkt->statp.st_mtime) {
102 Jmsg(jcr, M_SAVED, 0, _("%s st_mtime differs\n"), fname);
104 } else if (elt->ctime != ff_pkt->statp.st_ctime) {
105 Jmsg(jcr, M_SAVED, 0, _("%s st_ctime differs\n"), fname);
109 accurate_mark_file_as_seen(elt);
110 Dmsg2(500, "accurate %s = %i\n", fname, stat);
113 unstrip_path(ff_pkt);
118 * TODO: use bigbuffer from htable
120 int accurate_cmd(JCR *jcr)
122 BSOCK *dir = jcr->dir_bsock;
130 if (!jcr->accurate || job_canceled(jcr) || jcr->JobLevel==L_FULL) {
134 if (sscanf(dir->msg, "accurate files=%ld", &nb) != 1) {
135 dir->fsend(_("2991 Bad accurate command\n"));
138 Dmsg2(200, "nb=%d msg=%s\n", nb, dir->msg);
140 jcr->file_list = (htable *)malloc(sizeof(htable));
141 jcr->file_list->init(elt, &elt->link, nb);
144 * buffer = sizeof(CurFile) + dirmsg
145 * dirmsg = fname + \0 + lstat
147 /* get current files */
148 while (dir->recv() >= 0) {
149 len = strlen(dir->msg) + 1;
150 if (len < dir->msglen) {
151 /* we store CurFile, fname and ctime/mtime in the same chunk */
152 elt = (CurFile *)jcr->file_list->hash_malloc(sizeof(CurFile)+len);
153 elt->fname = (char *)elt+sizeof(CurFile);
154 strcpy(elt->fname, dir->msg);
155 lstat = dir->msg + len;
156 decode_stat(lstat, &statp, &LinkFIc); /* decode catalog stat */
157 elt->ctime = statp.st_ctime;
158 elt->mtime = statp.st_mtime;
160 jcr->file_list->insert(elt->fname, elt);
161 Dmsg2(500, "add fname=%s lstat=%s\n", elt->fname, lstat);
166 extern void *start_heap;
168 char b1[50], b2[50], b3[50], b4[50], b5[50];
169 Dmsg5(1," Heap: heap=%s smbytes=%s max_bytes=%s bufs=%s max_bufs=%s\n",
170 edit_uint64_with_commas((char *)sbrk(0)-(char *)start_heap, b1),
171 edit_uint64_with_commas(sm_bytes, b2),
172 edit_uint64_with_commas(sm_max_bytes, b3),
173 edit_uint64_with_commas(sm_buffers, b4),
174 edit_uint64_with_commas(sm_max_buffers, b5));
176 // jcr->file_list->stats();
182 bool accurate_send_deleted_list(JCR *jcr)
187 int stream = STREAM_UNIX_ATTRIBUTES;
189 if (!jcr->accurate || jcr->JobLevel == L_FULL) {
193 if (jcr->file_list == NULL) {
197 ff_pkt = init_find_files();
198 ff_pkt->type = FT_DELETED;
200 foreach_htable (elt, jcr->file_list) {
201 if (!accurate_file_has_been_seen(elt)) { /* already seen */
202 Dmsg2(500, "deleted fname=%s seen=%i\n", elt->fname, elt->seen);
203 ff_pkt->fname = elt->fname;
204 ff_pkt->statp.st_mtime = elt->mtime;
205 ff_pkt->statp.st_ctime = elt->ctime;
206 encode_and_send_attributes(jcr, ff_pkt, stream);
210 term_find_files(ff_pkt);
212 /* TODO: clean htable when this function is not reached ? */
213 if (jcr->file_list) {
214 jcr->file_list->destroy();
215 free(jcr->file_list);
216 jcr->file_list = NULL;
222 * check for BSD nodump flag
224 static bool no_dump(JCR *jcr, FF_PKT *ff_pkt)
226 #if defined(HAVE_CHFLAGS) && defined(UF_NODUMP)
227 if ( (ff_pkt->flags & FO_HONOR_NODUMP) &&
228 (ff_pkt->statp.st_flags & UF_NODUMP) ) {
229 Jmsg(jcr, M_INFO, 1, _(" NODUMP flag set - will not process %s\n"),
231 return true; /* do not backup this file */
234 return false; /* do backup */
238 * Find all the requested files and send them
239 * to the Storage daemon.
241 * Note, we normally carry on a one-way
242 * conversation from this point on with the SD, simply blasting
243 * data to him. To properly know what is going on, we
244 * also run a "heartbeat" monitor which reads the socket and
245 * reacts accordingly (at the moment it has nothing to do
246 * except echo the heartbeat to the Director).
249 bool blast_data_to_storage_daemon(JCR *jcr, char *addr)
253 // TODO landonf: Allow user to specify encryption algorithm
255 sd = jcr->store_bsock;
257 set_jcr_job_status(jcr, JS_Running);
259 Dmsg1(300, "bfiled: opened data connection %d to stored\n", sd->m_fd);
262 CLIENT *client = (CLIENT *)GetNextRes(R_CLIENT, NULL);
266 buf_size = client->max_network_buffer_size;
268 buf_size = 0; /* use default */
270 if (!sd->set_buffer_size(buf_size, BNET_SETBUF_WRITE)) {
271 set_jcr_job_status(jcr, JS_ErrorTerminated);
272 Jmsg(jcr, M_FATAL, 0, _("Cannot set buffer size FD->SD.\n"));
276 jcr->buf_size = sd->msglen;
277 /* Adjust for compression so that output buffer is
278 * 12 bytes + 0.1% larger than input buffer plus 18 bytes.
279 * This gives a bit extra plus room for the sparse addr if any.
280 * Note, we adjust the read size to be smaller so that the
281 * same output buffer can be used without growing it.
283 * The zlib compression workset is initialized here to minimize
284 * the "per file" load. The jcr member is only set, if the init
287 jcr->compress_buf_size = jcr->buf_size + ((jcr->buf_size+999) / 1000) + 30;
288 jcr->compress_buf = get_memory(jcr->compress_buf_size);
291 z_stream *pZlibStream = (z_stream*)malloc(sizeof(z_stream));
293 pZlibStream->zalloc = Z_NULL;
294 pZlibStream->zfree = Z_NULL;
295 pZlibStream->opaque = Z_NULL;
296 pZlibStream->state = Z_NULL;
298 if (deflateInit(pZlibStream, Z_DEFAULT_COMPRESSION) == Z_OK) {
299 jcr->pZLIB_compress_workset = pZlibStream;
306 if (!crypto_session_start(jcr)) {
310 set_find_options((FF_PKT *)jcr->ff, jcr->incremental, jcr->mtime);
312 /* in accurate mode, we overwrite the find_one check function */
314 set_find_changed_function((FF_PKT *)jcr->ff, accurate_check_file);
317 start_heartbeat_monitor(jcr);
319 jcr->acl_text = get_pool_memory(PM_MESSAGE);
321 /* Subroutine save_file() is called for each file */
322 if (!find_files(jcr, (FF_PKT *)jcr->ff, save_file, plugin_save)) {
323 ok = false; /* error */
324 set_jcr_job_status(jcr, JS_ErrorTerminated);
327 accurate_send_deleted_list(jcr); /* send deleted list to SD */
329 free_pool_memory(jcr->acl_text);
331 stop_heartbeat_monitor(jcr);
333 sd->signal(BNET_EOD); /* end of sending data */
339 if (jcr->compress_buf) {
340 free_pool_memory(jcr->compress_buf);
341 jcr->compress_buf = NULL;
343 if (jcr->pZLIB_compress_workset) {
344 /* Free the zlib stream */
346 deflateEnd((z_stream *)jcr->pZLIB_compress_workset);
348 free (jcr->pZLIB_compress_workset);
349 jcr->pZLIB_compress_workset = NULL;
351 crypto_session_end(jcr);
354 Dmsg1(100, "end blast_data ok=%d\n", ok);
358 static bool crypto_session_start(JCR *jcr)
360 crypto_cipher_t cipher = CRYPTO_CIPHER_AES_128_CBC;
363 * Create encryption session data and a cached, DER-encoded session data
364 * structure. We use a single session key for each backup, so we'll encode
365 * the session data only once.
367 if (jcr->crypto.pki_encrypt) {
370 /* Create per-job session encryption context */
371 jcr->crypto.pki_session = crypto_session_new(cipher, jcr->crypto.pki_recipients);
373 /* Get the session data size */
374 if (!crypto_session_encode(jcr->crypto.pki_session, (uint8_t *)0, &size)) {
375 Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n"));
379 /* Allocate buffer */
380 jcr->crypto.pki_session_encoded = get_memory(size);
382 /* Encode session data */
383 if (!crypto_session_encode(jcr->crypto.pki_session, (uint8_t *)jcr->crypto.pki_session_encoded, &size)) {
384 Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n"));
388 /* ... and store the encoded size */
389 jcr->crypto.pki_session_encoded_size = size;
391 /* Allocate the encryption/decryption buffer */
392 jcr->crypto.crypto_buf = get_memory(CRYPTO_CIPHER_MAX_BLOCK_SIZE);
397 static void crypto_session_end(JCR *jcr)
399 if (jcr->crypto.crypto_buf) {
400 free_pool_memory(jcr->crypto.crypto_buf);
401 jcr->crypto.crypto_buf = NULL;
403 if (jcr->crypto.pki_session) {
404 crypto_session_free(jcr->crypto.pki_session);
406 if (jcr->crypto.pki_session_encoded) {
407 free_pool_memory(jcr->crypto.pki_session_encoded);
408 jcr->crypto.pki_session_encoded = NULL;
412 static bool crypto_session_send(JCR *jcr, BSOCK *sd)
416 /* Send our header */
417 Dmsg2(100, "Send hdr fi=%ld stream=%d\n", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA);
418 sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA);
421 sd->msg = jcr->crypto.pki_session_encoded;
422 sd->msglen = jcr->crypto.pki_session_encoded_size;
423 jcr->JobBytes += sd->msglen;
425 Dmsg1(100, "Send data len=%d\n", sd->msglen);
428 sd->signal(BNET_EOD);
434 * Called here by find() for each file included.
435 * This is a callback. The original is find_files() above.
437 * Send the file and its data to the Storage daemon.
441 * -1 to ignore file/directory (not used here)
443 int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level)
445 bool do_read = false;
446 int stat, data_stream;
448 DIGEST *digest = NULL;
449 DIGEST *signing_digest = NULL;
450 int digest_stream = STREAM_NONE;
451 SIGNATURE *sig = NULL;
452 bool has_file_data = false;
453 // TODO landonf: Allow the user to specify the digest algorithm
455 crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA256;
457 crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA1;
459 BSOCK *sd = jcr->store_bsock;
461 if (job_canceled(jcr)) {
465 jcr->num_files_examined++; /* bump total file count */
467 switch (ff_pkt->type) {
468 case FT_LNKSAVED: /* Hard linked, file already saved */
469 Dmsg2(130, "FT_LNKSAVED hard link: %s => %s\n", ff_pkt->fname, ff_pkt->link);
472 Dmsg1(130, "FT_REGE saving: %s\n", ff_pkt->fname);
473 if (no_dump(jcr, ff_pkt))
475 has_file_data = true;
478 Dmsg1(130, "FT_REG saving: %s\n", ff_pkt->fname);
479 if (no_dump(jcr, ff_pkt))
481 has_file_data = true;
484 Dmsg2(130, "FT_LNK saving: %s -> %s\n", ff_pkt->fname, ff_pkt->link);
487 jcr->num_files_examined--; /* correct file count */
488 if (no_dump(jcr, ff_pkt)) /* disable recursion on nodump directories */
489 ff_pkt->flags |= FO_NO_RECURSION;
490 return 1; /* not used */
492 Jmsg(jcr, M_INFO, 1, _(" Recursion turned off. Will not descend from %s into %s\n"),
493 ff_pkt->top_fname, ff_pkt->fname);
494 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
497 /* Suppress message for /dev filesystems */
498 if (!is_in_fileset(ff_pkt)) {
499 Jmsg(jcr, M_INFO, 1, _(" %s is a different filesystem. Will not descend from %s into %s\n"),
500 ff_pkt->fname, ff_pkt->top_fname, ff_pkt->fname);
502 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
505 Jmsg(jcr, M_INFO, 1, _(" Disallowed filesystem. Will not descend from %s into %s\n"),
506 ff_pkt->top_fname, ff_pkt->fname);
507 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
510 Jmsg(jcr, M_INFO, 1, _(" Disallowed drive type. Will not descend into %s\n"),
515 Dmsg1(130, "FT_DIREND: %s\n", ff_pkt->link);
518 Dmsg1(130, "FT_SPEC saving: %s\n", ff_pkt->fname);
519 if (S_ISSOCK(ff_pkt->statp.st_mode)) {
520 Jmsg(jcr, M_SKIPPED, 1, _(" Socket file skipped: %s\n"), ff_pkt->fname);
525 Dmsg1(130, "FT_RAW saving: %s\n", ff_pkt->fname);
526 has_file_data = true;
529 Dmsg1(130, "FT_FIFO saving: %s\n", ff_pkt->fname);
533 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not access \"%s\": ERR=%s\n"), ff_pkt->fname,
534 be.bstrerror(ff_pkt->ff_errno));
540 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not follow link \"%s\": ERR=%s\n"),
541 ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
547 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not stat \"%s\": ERR=%s\n"), ff_pkt->fname,
548 be.bstrerror(ff_pkt->ff_errno));
554 Jmsg(jcr, M_SKIPPED, 1, _(" Unchanged file skipped: %s\n"), ff_pkt->fname);
557 Jmsg(jcr, M_NOTSAVED, 0, _(" Archive file not saved: %s\n"), ff_pkt->fname);
561 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not open directory \"%s\": ERR=%s\n"),
562 ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
567 Jmsg(jcr, M_NOTSAVED, 0, _(" Unknown file type %d; not saved: %s\n"),
568 ff_pkt->type, ff_pkt->fname);
573 Dmsg1(130, "bfiled: sending %s to stored\n", ff_pkt->fname);
575 /* Digests and encryption are only useful if there's file data */
578 * Setup for digest handling. If this fails, the digest will be set to NULL
579 * and not used. Note, the digest (file hash) can be any one of the four
582 * The signing digest is a single algorithm depending on
583 * whether or not we have SHA2.
584 * ****FIXME**** the signing algoritm should really be
585 * determined a different way!!!!!! What happens if
586 * sha2 was available during backup but not restore?
588 if (ff_pkt->flags & FO_MD5) {
589 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_MD5);
590 digest_stream = STREAM_MD5_DIGEST;
592 } else if (ff_pkt->flags & FO_SHA1) {
593 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA1);
594 digest_stream = STREAM_SHA1_DIGEST;
596 } else if (ff_pkt->flags & FO_SHA256) {
597 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA256);
598 digest_stream = STREAM_SHA256_DIGEST;
600 } else if (ff_pkt->flags & FO_SHA512) {
601 digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA512);
602 digest_stream = STREAM_SHA512_DIGEST;
605 /* Did digest initialization fail? */
606 if (digest_stream != STREAM_NONE && digest == NULL) {
607 Jmsg(jcr, M_WARNING, 0, _("%s digest initialization failed\n"),
608 stream_to_ascii(digest_stream));
612 * Set up signature digest handling. If this fails, the signature digest will be set to
615 // TODO landonf: We should really only calculate the digest once, for both verification and signing.
616 if (jcr->crypto.pki_sign) {
617 signing_digest = crypto_digest_new(jcr, signing_algorithm);
619 /* Full-stop if a failure occurred initializing the signature digest */
620 if (signing_digest == NULL) {
621 Jmsg(jcr, M_NOTSAVED, 0, _("%s signature digest initialization failed\n"),
622 stream_to_ascii(signing_algorithm));
628 /* Enable encryption */
629 if (jcr->crypto.pki_encrypt) {
630 ff_pkt->flags |= FO_ENCRYPT;
634 /* Initialize the file descriptor we use for data and other streams. */
636 if (ff_pkt->flags & FO_PORTABLE) {
637 set_portable_backup(&ff_pkt->bfd); /* disable Win32 BackupRead() */
639 if (ff_pkt->cmd_plugin) {
640 if (!set_cmd_plugin(&ff_pkt->bfd, jcr)) {
643 send_plugin_name(jcr, sd, true); /* signal start of plugin data */
646 /* Send attributes -- must be done after binit() */
647 if (!encode_and_send_attributes(jcr, ff_pkt, data_stream)) {
651 /* Set up the encryption context and send the session data to the SD */
652 if (has_file_data && jcr->crypto.pki_encrypt) {
653 if (!crypto_session_send(jcr, sd)) {
659 * Open any file with data that we intend to save, then save it.
661 * Note, if is_win32_backup, we must open the Directory so that
662 * the BackupRead will save its permissions and ownership streams.
664 if (ff_pkt->type != FT_LNKSAVED && S_ISREG(ff_pkt->statp.st_mode)) {
666 do_read = !is_portable_backup(&ff_pkt->bfd) || ff_pkt->statp.st_size > 0;
668 do_read = ff_pkt->statp.st_size > 0;
670 } else if (ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO ||
671 ff_pkt->type == FT_REPARSE ||
672 (!is_portable_backup(&ff_pkt->bfd) && ff_pkt->type == FT_DIREND)) {
675 if (ff_pkt->cmd_plugin) {
679 Dmsg1(100, "do_read=%d\n", do_read);
683 if (ff_pkt->type == FT_FIFO) {
684 tid = start_thread_timer(jcr, pthread_self(), 60);
688 int noatime = ff_pkt->flags & FO_NOATIME ? O_NOATIME : 0;
689 ff_pkt->bfd.reparse_point = ff_pkt->type == FT_REPARSE;
690 if (bopen(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY | noatime, 0) < 0) {
691 ff_pkt->ff_errno = errno;
693 Jmsg(jcr, M_NOTSAVED, 0, _(" Cannot open \"%s\": ERR=%s.\n"), ff_pkt->fname,
697 stop_thread_timer(tid);
703 stop_thread_timer(tid);
707 stat = send_data(jcr, data_stream, ff_pkt, digest, signing_digest);
709 if (ff_pkt->flags & FO_CHKCHANGES) {
710 has_file_changed(jcr, ff_pkt);
713 bclose(&ff_pkt->bfd);
720 #ifdef HAVE_DARWIN_OS
721 /* Regular files can have resource forks and Finder Info */
722 if (ff_pkt->type != FT_LNKSAVED && (S_ISREG(ff_pkt->statp.st_mode) &&
723 ff_pkt->flags & FO_HFSPLUS)) {
724 if (ff_pkt->hfsinfo.rsrclength > 0) {
727 if (!bopen_rsrc(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0) < 0) {
728 ff_pkt->ff_errno = errno;
730 Jmsg(jcr, M_NOTSAVED, -1, _(" Cannot open resource fork for \"%s\": ERR=%s.\n"),
731 ff_pkt->fname, be.bstrerror());
733 if (is_bopen(&ff_pkt->bfd)) {
734 bclose(&ff_pkt->bfd);
738 flags = ff_pkt->flags;
739 ff_pkt->flags &= ~(FO_GZIP|FO_SPARSE);
740 if (flags & FO_ENCRYPT) {
741 rsrc_stream = STREAM_ENCRYPTED_MACOS_FORK_DATA;
743 rsrc_stream = STREAM_MACOS_FORK_DATA;
745 stat = send_data(jcr, rsrc_stream, ff_pkt, digest, signing_digest);
746 ff_pkt->flags = flags;
747 bclose(&ff_pkt->bfd);
753 Dmsg1(300, "Saving Finder Info for \"%s\"\n", ff_pkt->fname);
754 sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_HFSPLUS_ATTRIBUTES);
755 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
756 memcpy(sd->msg, ff_pkt->hfsinfo.fndrinfo, 32);
759 crypto_digest_update(digest, (uint8_t *)sd->msg, sd->msglen);
761 if (signing_digest) {
762 crypto_digest_update(signing_digest, (uint8_t *)sd->msg, sd->msglen);
765 sd->signal(BNET_EOD);
769 if (ff_pkt->flags & FO_ACL) {
770 /* Read access ACLs for files, dirs and links */
771 if (!read_and_send_acl(jcr, BACL_TYPE_ACCESS, STREAM_UNIX_ACCESS_ACL)) {
774 /* Directories can have default ACLs too */
775 if (ff_pkt->type == FT_DIREND && (BACL_CAP & BACL_CAP_DEFAULTS_DIR)) {
776 if (!read_and_send_acl(jcr, BACL_TYPE_DEFAULT, STREAM_UNIX_DEFAULT_ACL)) {
782 /* Terminate the signing digest and send it to the Storage daemon */
783 if (signing_digest) {
786 if ((sig = crypto_sign_new(jcr)) == NULL) {
787 Jmsg(jcr, M_FATAL, 0, _("Failed to allocate memory for crypto signature.\n"));
791 if (!crypto_sign_add_signer(sig, signing_digest, jcr->crypto.pki_keypair)) {
792 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
796 /* Get signature size */
797 if (!crypto_sign_encode(sig, NULL, &size)) {
798 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
802 /* Grow the bsock buffer to fit our message if necessary */
803 if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
804 sd->msg = realloc_pool_memory(sd->msg, size);
807 /* Send our header */
808 sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_SIGNED_DIGEST);
809 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
811 /* Encode signature data */
812 if (!crypto_sign_encode(sig, (uint8_t *)sd->msg, &size)) {
813 Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
819 sd->signal(BNET_EOD); /* end of checksum */
822 /* Terminate any digest and send it to Storage daemon */
826 sd->fsend("%ld %d 0", jcr->JobFiles, digest_stream);
827 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
829 size = CRYPTO_DIGEST_MAX_SIZE;
831 /* Grow the bsock buffer to fit our message if necessary */
832 if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
833 sd->msg = realloc_pool_memory(sd->msg, size);
836 if (!crypto_digest_finalize(digest, (uint8_t *)sd->msg, &size)) {
837 Jmsg(jcr, M_FATAL, 0, _("An error occurred finalizing signing the stream.\n"));
843 sd->signal(BNET_EOD); /* end of checksum */
845 if (ff_pkt->cmd_plugin) {
846 send_plugin_name(jcr, sd, false); /* signal end of plugin data */
850 rtnstat = 1; /* good return */
854 crypto_digest_free(digest);
856 if (signing_digest) {
857 crypto_digest_free(signing_digest);
860 crypto_sign_free(sig);
866 * Send data read from an already open file descriptor.
868 * We return 1 on sucess and 0 on errors.
871 * We use ff_pkt->statp.st_size when FO_SPARSE to know when to stop
873 * Currently this is not a problem as the only other stream, resource forks,
874 * are not handled as sparse files.
876 int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest,
877 DIGEST *signing_digest)
879 BSOCK *sd = jcr->store_bsock;
880 uint64_t fileAddr = 0; /* file address */
882 int32_t rsize = jcr->buf_size; /* read buffer size */
884 CIPHER_CONTEXT *cipher_ctx = NULL; /* Quell bogus uninitialized warnings */
885 const uint8_t *cipher_input;
886 uint32_t cipher_input_len;
887 uint32_t cipher_block_size;
888 uint32_t encrypted_len;
889 #ifdef FD_NO_SEND_TEST
894 rbuf = sd->msg; /* read buffer */
895 wbuf = sd->msg; /* write buffer */
896 cipher_input = (uint8_t *)rbuf; /* encrypt uncompressed data */
898 Dmsg1(300, "Saving data, type=%d\n", ff_pkt->type);
901 uLong compress_len = 0;
902 uLong max_compress_len = 0;
903 const Bytef *cbuf = NULL;
906 if (ff_pkt->flags & FO_GZIP) {
907 if (ff_pkt->flags & FO_SPARSE) {
908 cbuf = (Bytef *)jcr->compress_buf + SPARSE_FADDR_SIZE;
909 max_compress_len = jcr->compress_buf_size - SPARSE_FADDR_SIZE;
911 cbuf = (Bytef *)jcr->compress_buf;
912 max_compress_len = jcr->compress_buf_size; /* set max length */
914 wbuf = jcr->compress_buf; /* compressed output here */
915 cipher_input = (uint8_t *)jcr->compress_buf; /* encrypt compressed data */
918 * Only change zlib parameters if there is no pending operation.
919 * This should never happen as deflatereset is called after each
923 if (((z_stream*)jcr->pZLIB_compress_workset)->total_in == 0) {
924 /* set gzip compression level - must be done per file */
925 if ((zstat=deflateParams((z_stream*)jcr->pZLIB_compress_workset,
926 ff_pkt->GZIP_level, Z_DEFAULT_STRATEGY)) != Z_OK) {
927 Jmsg(jcr, M_FATAL, 0, _("Compression deflateParams error: %d\n"), zstat);
928 set_jcr_job_status(jcr, JS_ErrorTerminated);
934 const uint32_t max_compress_len = 0;
937 if (ff_pkt->flags & FO_ENCRYPT) {
938 if (ff_pkt->flags & FO_SPARSE) {
939 Jmsg0(jcr, M_FATAL, 0, _("Encrypting sparse data not supported.\n"));
942 /* Allocate the cipher context */
943 if ((cipher_ctx = crypto_cipher_new(jcr->crypto.pki_session, true,
944 &cipher_block_size)) == NULL) {
945 /* Shouldn't happen! */
946 Jmsg0(jcr, M_FATAL, 0, _("Failed to initialize encryption context.\n"));
951 * Grow the crypto buffer, if necessary.
952 * crypto_cipher_update() will buffer up to (cipher_block_size - 1).
953 * We grow crypto_buf to the maximum number of blocks that
954 * could be returned for the given read buffer size.
955 * (Using the larger of either rsize or max_compress_len)
957 jcr->crypto.crypto_buf = check_pool_memory_size(jcr->crypto.crypto_buf,
958 (MAX(rsize + (int)sizeof(uint32_t), (int32_t)max_compress_len) +
959 cipher_block_size - 1) / cipher_block_size * cipher_block_size);
961 wbuf = jcr->crypto.crypto_buf; /* Encrypted, possibly compressed output here. */
965 * Send Data header to Storage daemon
966 * <file-index> <stream> <info>
968 if (!sd->fsend("%ld %d 0", jcr->JobFiles, stream)) {
969 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
973 Dmsg1(300, ">stored: datahdr %s\n", sd->msg);
976 * Make space at beginning of buffer for fileAddr because this
977 * same buffer will be used for writing if compression is off.
979 if (ff_pkt->flags & FO_SPARSE) {
980 rbuf += SPARSE_FADDR_SIZE;
981 rsize -= SPARSE_FADDR_SIZE;
982 #ifdef HAVE_FREEBSD_OS
984 * To read FreeBSD partitions, the read size must be
987 rsize = (rsize/512) * 512;
991 /* a RAW device read on win32 only works if the buffer is a multiple of 512 */
993 if (S_ISBLK(ff_pkt->statp.st_mode))
994 rsize = (rsize/512) * 512;
1000 while ((sd->msglen=(uint32_t)bread(&ff_pkt->bfd, rbuf, rsize)) > 0) {
1002 /* Check for sparse blocks */
1003 if (ff_pkt->flags & FO_SPARSE) {
1005 bool haveBlock = true;
1006 if (sd->msglen == rsize &&
1007 fileAddr+sd->msglen < (uint64_t)ff_pkt->statp.st_size ||
1008 ((ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO) &&
1009 (uint64_t)ff_pkt->statp.st_size == 0)) {
1010 haveBlock = !is_buf_zero(rbuf, rsize);
1013 ser_begin(wbuf, SPARSE_FADDR_SIZE);
1014 ser_uint64(fileAddr); /* store fileAddr in begin of buffer */
1016 fileAddr += sd->msglen; /* update file address */
1018 continue; /* skip block of zeros */
1022 jcr->ReadBytes += sd->msglen; /* count bytes read */
1024 /* Uncompressed cipher input length */
1025 cipher_input_len = sd->msglen;
1027 /* Update checksum if requested */
1029 crypto_digest_update(digest, (uint8_t *)rbuf, sd->msglen);
1032 /* Update signing digest if requested */
1033 if (signing_digest) {
1034 crypto_digest_update(signing_digest, (uint8_t *)rbuf, sd->msglen);
1038 /* Do compression if turned on */
1039 if (ff_pkt->flags & FO_GZIP && jcr->pZLIB_compress_workset) {
1040 Dmsg3(400, "cbuf=0x%x rbuf=0x%x len=%u\n", cbuf, rbuf, sd->msglen);
1042 ((z_stream*)jcr->pZLIB_compress_workset)->next_in = (Bytef *)rbuf;
1043 ((z_stream*)jcr->pZLIB_compress_workset)->avail_in = sd->msglen;
1044 ((z_stream*)jcr->pZLIB_compress_workset)->next_out = (Bytef *)cbuf;
1045 ((z_stream*)jcr->pZLIB_compress_workset)->avail_out = max_compress_len;
1047 if ((zstat=deflate((z_stream*)jcr->pZLIB_compress_workset, Z_FINISH)) != Z_STREAM_END) {
1048 Jmsg(jcr, M_FATAL, 0, _("Compression deflate error: %d\n"), zstat);
1049 set_jcr_job_status(jcr, JS_ErrorTerminated);
1052 compress_len = ((z_stream*)jcr->pZLIB_compress_workset)->total_out;
1053 /* reset zlib stream to be able to begin from scratch again */
1054 if ((zstat=deflateReset((z_stream*)jcr->pZLIB_compress_workset)) != Z_OK) {
1055 Jmsg(jcr, M_FATAL, 0, _("Compression deflateReset error: %d\n"), zstat);
1056 set_jcr_job_status(jcr, JS_ErrorTerminated);
1060 Dmsg2(400, "compressed len=%d uncompressed len=%d\n", compress_len,
1063 sd->msglen = compress_len; /* set compressed length */
1064 cipher_input_len = compress_len;
1068 * Note, here we prepend the current record length to the beginning
1069 * of the encrypted data. This is because both sparse and compression
1070 * restore handling want records returned to them with exactly the
1071 * same number of bytes that were processed in the backup handling.
1072 * That is, both are block filters rather than a stream. When doing
1073 * compression, the compression routines may buffer data, so that for
1074 * any one record compressed, when it is decompressed the same size
1075 * will not be obtained. Of course, the buffered data eventually comes
1076 * out in subsequent crypto_cipher_update() calls or at least
1077 * when crypto_cipher_finalize() is called. Unfortunately, this
1078 * "feature" of encryption enormously complicates the restore code.
1080 if (ff_pkt->flags & FO_ENCRYPT) {
1081 uint32_t initial_len = 0;
1084 if (ff_pkt->flags & FO_SPARSE) {
1085 cipher_input_len += SPARSE_FADDR_SIZE;
1088 /* Encrypt the length of the input block */
1089 uint8_t packet_len[sizeof(uint32_t)];
1091 ser_begin(packet_len, sizeof(uint32_t));
1092 ser_uint32(cipher_input_len); /* store data len in begin of buffer */
1093 Dmsg1(20, "Encrypt len=%d\n", cipher_input_len);
1095 if (!crypto_cipher_update(cipher_ctx, packet_len, sizeof(packet_len),
1096 (uint8_t *)jcr->crypto.crypto_buf, &initial_len)) {
1097 /* Encryption failed. Shouldn't happen. */
1098 Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
1102 /* Encrypt the input block */
1103 if (crypto_cipher_update(cipher_ctx, cipher_input, cipher_input_len,
1104 (uint8_t *)&jcr->crypto.crypto_buf[initial_len], &encrypted_len)) {
1105 if ((initial_len + encrypted_len) == 0) {
1106 /* No full block of data available, read more data */
1109 Dmsg2(400, "encrypted len=%d unencrypted len=%d\n", encrypted_len,
1111 sd->msglen = initial_len + encrypted_len; /* set encrypted length */
1113 /* Encryption failed. Shouldn't happen. */
1114 Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
1119 /* Send the buffer to the Storage daemon */
1120 if (ff_pkt->flags & FO_SPARSE) {
1121 sd->msglen += SPARSE_FADDR_SIZE; /* include fileAddr in size */
1123 sd->msg = wbuf; /* set correct write buffer */
1125 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1129 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
1131 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
1132 sd->msg = msgsave; /* restore read buffer */
1134 } /* end while read file data */
1136 if (sd->msglen < 0) { /* error */
1138 Jmsg(jcr, M_ERROR, 0, _("Read error on file %s. ERR=%s\n"),
1139 ff_pkt->fname, be.bstrerror(ff_pkt->bfd.berrno));
1140 if (jcr->Errors++ > 1000) { /* insanity check */
1141 Jmsg(jcr, M_FATAL, 0, _("Too many errors.\n"));
1143 } else if (ff_pkt->flags & FO_ENCRYPT) {
1145 * For encryption, we must call finalize to push out any
1148 if (!crypto_cipher_finalize(cipher_ctx, (uint8_t *)jcr->crypto.crypto_buf,
1150 /* Padding failed. Shouldn't happen. */
1151 Jmsg(jcr, M_FATAL, 0, _("Encryption padding error\n"));
1155 /* Note, on SSL pre-0.9.7, there is always some output */
1156 if (encrypted_len > 0) {
1157 sd->msglen = encrypted_len; /* set encrypted length */
1158 sd->msg = jcr->crypto.crypto_buf; /* set correct write buffer */
1160 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1164 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
1165 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
1166 sd->msg = msgsave; /* restore bnet buffer */
1170 if (!sd->signal(BNET_EOD)) { /* indicate end of file data */
1171 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1176 /* Free the cipher context */
1178 crypto_cipher_free(cipher_ctx);
1183 /* Free the cipher context */
1185 crypto_cipher_free(cipher_ctx);
1188 sd->msg = msgsave; /* restore bnet buffer */
1194 * Read and send an ACL for the last encountered file.
1196 static bool read_and_send_acl(JCR *jcr, int acltype, int stream)
1199 BSOCK *sd = jcr->store_bsock;
1202 #ifdef FD_NO_SEND_TEST
1206 len = bacl_get(jcr, acltype);
1208 Jmsg1(jcr, M_WARNING, 0, _("Error reading ACL of %s\n"), jcr->last_fname);
1212 return true; /* no ACL */
1216 if (!sd->fsend("%ld %d 0", jcr->JobFiles, stream)) {
1217 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1222 /* Send the buffer to the storage deamon */
1223 Dmsg2(400, "Backing up ACL type 0x%2x <%s>\n", acltype, jcr->acl_text);
1225 sd->msg = jcr->acl_text;
1226 sd->msglen = len + 1;
1230 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1235 jcr->JobBytes += sd->msglen;
1237 if (!sd->signal(BNET_EOD)) {
1238 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1243 Dmsg1(200, "ACL of file: %s successfully backed up!\n", jcr->last_fname);
1248 static bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream)
1250 BSOCK *sd = jcr->store_bsock;
1251 char attribs[MAXSTRING];
1252 char attribsEx[MAXSTRING];
1255 #ifdef FD_NO_SEND_TEST
1259 Dmsg1(300, "encode_and_send_attrs fname=%s\n", ff_pkt->fname);
1260 /* Find what data stream we will use, then encode the attributes */
1261 if ((data_stream = select_data_stream(ff_pkt)) == STREAM_NONE) {
1262 /* This should not happen */
1263 Jmsg0(jcr, M_FATAL, 0, _("Invalid file flags, no supported data stream type.\n"));
1266 encode_stat(attribs, ff_pkt, data_stream);
1268 /* Now possibly extend the attributes */
1269 attr_stream = encode_attribsEx(jcr, attribsEx, ff_pkt);
1271 Dmsg3(300, "File %s\nattribs=%s\nattribsEx=%s\n", ff_pkt->fname, attribs, attribsEx);
1274 jcr->JobFiles++; /* increment number of files sent */
1275 ff_pkt->FileIndex = jcr->JobFiles; /* return FileIndex */
1276 pm_strcpy(jcr->last_fname, ff_pkt->fname);
1280 * Send Attributes header to Storage daemon
1281 * <file-index> <stream> <info>
1283 if (!sd->fsend("%ld %d 0", jcr->JobFiles, attr_stream)) {
1284 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1288 Dmsg1(300, ">stored: attrhdr %s\n", sd->msg);
1291 * Send file attributes to Storage daemon
1294 * Filename (full path)
1295 * Encoded attributes
1296 * Link name (if type==FT_LNK or FT_LNKSAVED)
1297 * Encoded extended-attributes (for Win32)
1299 * For a directory, link is the same as fname, but with trailing
1300 * slash. For a linked file, link is the link.
1302 if (ff_pkt->type != FT_DELETED) { /* already stripped */
1305 if (ff_pkt->type == FT_LNK || ff_pkt->type == FT_LNKSAVED) {
1306 Dmsg2(300, "Link %s to %s\n", ff_pkt->fname, ff_pkt->link);
1307 stat = sd->fsend("%ld %d %s%c%s%c%s%c%s%c", jcr->JobFiles,
1308 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, ff_pkt->link, 0,
1310 } else if (ff_pkt->type == FT_DIREND || ff_pkt->type == FT_REPARSE) {
1311 /* Here link is the canonical filename (i.e. with trailing slash) */
1312 stat = sd->fsend("%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
1313 ff_pkt->type, ff_pkt->link, 0, attribs, 0, 0, attribsEx, 0);
1315 stat = sd->fsend("%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
1316 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, 0, attribsEx, 0);
1318 if (ff_pkt->type != FT_DELETED) {
1319 unstrip_path(ff_pkt);
1322 Dmsg2(300, ">stored: attr len=%d: %s\n", sd->msglen, sd->msg);
1324 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
1328 sd->signal(BNET_EOD); /* indicate end of attributes data */
1333 * Do in place strip of path
1335 static bool do_strip(int count, char *in)
1341 /* Copy to first path separator -- Win32 might have c: ... */
1342 while (*in && !IsPathSeparator(*in)) {
1346 numsep++; /* one separator seen */
1347 for (stripped=0; stripped<count && *in; stripped++) {
1348 while (*in && !IsPathSeparator(*in)) {
1349 in++; /* skip chars */
1352 numsep++; /* count separators seen */
1353 in++; /* skip separator */
1357 while (*in) { /* copy to end */
1358 if (IsPathSeparator(*in)) {
1364 Dmsg4(500, "stripped=%d count=%d numsep=%d sep>count=%d\n",
1365 stripped, count, numsep, numsep>count);
1366 return stripped==count && numsep>count;
1370 * If requested strip leading components of the path so that we can
1371 * save file as if it came from a subdirectory. This is most useful
1372 * for dealing with snapshots, by removing the snapshot directory, or
1373 * in handling vendor migrations where files have been restored with
1374 * a vendor product into a subdirectory.
1376 static void strip_path(FF_PKT *ff_pkt)
1378 if (!(ff_pkt->flags & FO_STRIPPATH) || ff_pkt->strip_path <= 0) {
1379 Dmsg1(200, "No strip for %s\n", ff_pkt->fname);
1382 if (!ff_pkt->fname_save) {
1383 ff_pkt->fname_save = get_pool_memory(PM_FNAME);
1384 ff_pkt->link_save = get_pool_memory(PM_FNAME);
1386 pm_strcpy(ff_pkt->fname_save, ff_pkt->fname);
1389 * Strip path. If it doesn't succeed put it back. If
1390 * it does, and there is a different link string,
1391 * attempt to strip the link. If it fails, back them
1393 * Do not strip symlinks.
1394 * I.e. if either stripping fails don't strip anything.
1396 if (do_strip(ff_pkt->strip_path, ff_pkt->fname)) {
1397 /* Strip links but not symlinks */
1398 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1399 pm_strcpy(ff_pkt->link_save, ff_pkt->link);
1400 if (!do_strip(ff_pkt->strip_path, ff_pkt->link)) {
1401 strcpy(ff_pkt->link, ff_pkt->link_save);
1402 strcpy(ff_pkt->fname, ff_pkt->fname_save);
1406 strcpy(ff_pkt->fname, ff_pkt->fname_save);
1408 Dmsg2(200, "fname=%s stripped=%s\n", ff_pkt->fname_save, ff_pkt->fname);
1411 static void unstrip_path(FF_PKT *ff_pkt)
1413 if (!(ff_pkt->flags & FO_STRIPPATH) || ff_pkt->strip_path <= 0) {
1416 strcpy(ff_pkt->fname, ff_pkt->fname_save);
1417 if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
1418 strcpy(ff_pkt->link, ff_pkt->link_save);